diff --git a/.gitattributes b/.gitattributes index 19503c8a7d3b981db93df8beb4f3d73a172fe0e8..7e9c1d9d3dd2f9a67cfb8e26cbf10baf33749e2e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -245,3 +245,897 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text 0219155836_cosmos_tnt_spatial/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 filter=lfs diff=lfs merge=lfs -text 0219155836_cosmos_tnt_spatial/videos/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef.mp4 filter=lfs diff=lfs merge=lfs -text 0219155836_cosmos_tnt_spatial/videos/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5.mp4 filter=lfs diff=lfs merge=lfs -text +0219194937_cosmos_minibench/videos/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23.lock b/0219194937_cosmos_minibench/.locks/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580.lock b/0219194937_cosmos_minibench/.locks/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d.lock b/0219194937_cosmos_minibench/.locks/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c.lock new file mode 100644 index 0000000000000000000000000000000000000000..0c7fa61001ed7d44b42702dba672ef2c96b38f97 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659282 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8.lock new file mode 100644 index 0000000000000000000000000000000000000000..9761c8ed55eb6e5f4f52031a9ff6740b04e44cad --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659281 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960.lock new file mode 100644 index 0000000000000000000000000000000000000000..5b0d2cab974009b68f6afe0bca4315ae2551557b --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659280 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e.lock new file mode 100644 index 0000000000000000000000000000000000000000..03daf035ce5f6fae60c9a173c2fc18c405e659db --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659277 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d.lock new file mode 100644 index 0000000000000000000000000000000000000000..1501d02349ac574100c78adda453cd19d5317889 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659279 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168.lock new file mode 100644 index 0000000000000000000000000000000000000000..c908b4522965ba4ec3c1054a10981384df8fd128 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659283 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/.locks/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d.lock b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d.lock new file mode 100644 index 0000000000000000000000000000000000000000..0d62311c0553a3864f409ac0af7356e8064fb7c2 --- /dev/null +++ b/0219194937_cosmos_minibench/.locks/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2659278 \ No newline at end of file diff --git a/0219194937_cosmos_minibench/gpu_0.log b/0219194937_cosmos_minibench/gpu_0.log new file mode 100644 index 0000000000000000000000000000000000000000..0449eb4350ab528845dd05f3deff385a26675bae --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_0.log @@ -0,0 +1,46 @@ +[lock] Claimed: mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168 +[lock] Claimed: mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41 +[lock] Claimed: mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03 +[lock] Claimed: mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9 +[lock] Claimed: mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d +[lock] Claimed: mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97 +[lock] Claimed: mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c +[lock] Claimed: mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562 +[lock] Claimed: mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca +[lock] Claimed: mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50 +[lock] Claimed: mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068 +[lock] Claimed: mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89 +[lock] Claimed: mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02 +[lock] Claimed: mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0 +[lock] Claimed: mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15 +[lock] Claimed: mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be +[lock] Claimed: mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0 +[lock] Claimed: mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f +[lock] Claimed: mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940 +[lock] Claimed: mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872 +[lock] Claimed: mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455 +[lock] Claimed: mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3 +[lock] Claimed: mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287 (worker=bravo-tongclass-a800:2659283) +[lock] Done: mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287 diff --git a/0219194937_cosmos_minibench/gpu_0_instances.txt b/0219194937_cosmos_minibench/gpu_0_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..27f7a63e96be60430c68c99fae2f7a0ca72b0981 --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_0_instances.txt @@ -0,0 +1,23 @@ +mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168 +mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41 +mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03 +mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9 +mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d +mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97 +mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c +mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562 +mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca +mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50 +mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068 +mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89 +mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02 +mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0 +mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15 +mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be +mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0 +mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f +mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940 +mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872 +mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455 +mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3 +mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287 diff --git a/0219194937_cosmos_minibench/gpu_1.log b/0219194937_cosmos_minibench/gpu_1.log new file mode 100644 index 0000000000000000000000000000000000000000..7627af971e83b00a853baa8caf3c518360308a90 --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_1.log @@ -0,0 +1,46 @@ +[lock] Claimed: mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66 +[lock] Claimed: mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18 +[lock] Claimed: mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d +[lock] Claimed: mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2 +[lock] Claimed: mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23 +[lock] Claimed: mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc +[lock] Claimed: mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f +[lock] Claimed: mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7 +[lock] Claimed: mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459 +[lock] Claimed: mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f +[lock] Claimed: mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8 +[lock] Claimed: mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709 +[lock] Claimed: mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf +[lock] Claimed: mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2 +[lock] Claimed: mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5 +[lock] Claimed: mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e +[lock] Claimed: mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0 +[lock] Claimed: mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89 +[lock] Claimed: mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2 +[lock] Claimed: mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd +[lock] Claimed: mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206 +[lock] Claimed: mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f +[lock] Claimed: mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540 (worker=bravo-tongclass-a800:2659281) +[lock] Done: mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540 diff --git a/0219194937_cosmos_minibench/gpu_1_instances.txt b/0219194937_cosmos_minibench/gpu_1_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..2a54b851f5f276c1ab3221fe8e5a12ec08409b5a --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_1_instances.txt @@ -0,0 +1,23 @@ +mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66 +mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18 +mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d +mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2 +mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23 +mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc +mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f +mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7 +mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459 +mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f +mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8 +mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709 +mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf +mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2 +mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5 +mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e +mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0 +mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89 +mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2 +mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd +mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206 +mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f +mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540 diff --git a/0219194937_cosmos_minibench/gpu_2.log b/0219194937_cosmos_minibench/gpu_2.log new file mode 100644 index 0000000000000000000000000000000000000000..0ee71a5a9e4c17405323c30fb2e90f06b922a3dd --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_2.log @@ -0,0 +1,46 @@ +[lock] Claimed: mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c +[lock] Claimed: mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1 +[lock] Claimed: mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3 +[lock] Claimed: mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959 +[lock] Claimed: mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622 +[lock] Claimed: mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8 +[lock] Claimed: mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e +[lock] Claimed: mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c +[lock] Claimed: mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4 +[lock] Claimed: mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c +[lock] Claimed: mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b +[lock] Claimed: mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc +[lock] Claimed: mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d +[lock] Claimed: mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b +[lock] Claimed: mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25 +[lock] Claimed: mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d +[lock] Claimed: mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b +[lock] Claimed: mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e +[lock] Claimed: mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac +[lock] Claimed: mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8 +[lock] Claimed: mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4 +[lock] Claimed: mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067 (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067 +[lock] Claimed: mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d (worker=bravo-tongclass-a800:2659282) +[lock] Done: mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d diff --git a/0219194937_cosmos_minibench/gpu_2_instances.txt b/0219194937_cosmos_minibench/gpu_2_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..aef56ea1b1f4684bd1d234e57cc5973a253a6862 --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_2_instances.txt @@ -0,0 +1,23 @@ +mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c +mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1 +mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3 +mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959 +mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622 +mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8 +mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e +mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c +mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4 +mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c +mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b +mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc +mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d +mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b +mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25 +mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d +mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b +mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e +mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac +mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8 +mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4 +mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067 +mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d diff --git a/0219194937_cosmos_minibench/gpu_3.log b/0219194937_cosmos_minibench/gpu_3.log new file mode 100644 index 0000000000000000000000000000000000000000..aefe4a3c67d8b9e7155abe6fad97923d7bbdb10b --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_3.log @@ -0,0 +1,46 @@ +[lock] Claimed: mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0 +[lock] Claimed: mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f +[lock] Claimed: mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f +[lock] Claimed: mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc +[lock] Claimed: mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580 +[lock] Claimed: mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0 +[lock] Claimed: mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164 +[lock] Claimed: mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb +[lock] Claimed: mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe +[lock] Claimed: mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104 +[lock] Claimed: mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a +[lock] Claimed: mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32 +[lock] Claimed: mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960 +[lock] Claimed: mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706 +[lock] Claimed: mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5 +[lock] Claimed: mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3 +[lock] Claimed: mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325 +[lock] Claimed: mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807 +[lock] Claimed: mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8 +[lock] Claimed: mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b +[lock] Claimed: mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336 +[lock] Claimed: mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a +[lock] Claimed: mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093 (worker=bravo-tongclass-a800:2659280) +[lock] Done: mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093 diff --git a/0219194937_cosmos_minibench/gpu_3_instances.txt b/0219194937_cosmos_minibench/gpu_3_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..cd0ae906f5c55c5d13bd4f83fd68d951c98d06cf --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_3_instances.txt @@ -0,0 +1,23 @@ +mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0 +mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f +mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f +mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc +mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580 +mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0 +mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164 +mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb +mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe +mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104 +mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a +mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32 +mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960 +mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706 +mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5 +mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3 +mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325 +mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807 +mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8 +mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b +mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336 +mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a +mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093 diff --git a/0219194937_cosmos_minibench/gpu_4.log b/0219194937_cosmos_minibench/gpu_4.log new file mode 100644 index 0000000000000000000000000000000000000000..9f9c77d276be2649a25090c3315f346079ac17e6 --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_4.log @@ -0,0 +1,46 @@ +[lock] Claimed: mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45 +[lock] Claimed: mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529 +[lock] Claimed: mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d +[lock] Claimed: mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be +[lock] Claimed: mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb +[lock] Claimed: mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d +[lock] Claimed: mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d +[lock] Claimed: mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d +[lock] Claimed: mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a +[lock] Claimed: mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328 +[lock] Claimed: mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52 +[lock] Claimed: mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6 +[lock] Claimed: mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5 +[lock] Claimed: mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b +[lock] Claimed: mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167 +[lock] Claimed: mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632 +[lock] Claimed: mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c +[lock] Claimed: mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2 +[lock] Claimed: mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53 +[lock] Claimed: mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80 +[lock] Claimed: mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183 +[lock] Claimed: mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129 +[lock] Claimed: mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7 (worker=bravo-tongclass-a800:2659278) +[lock] Done: mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7 diff --git a/0219194937_cosmos_minibench/gpu_4_instances.txt b/0219194937_cosmos_minibench/gpu_4_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..e852a705ad71c7a76922c869e9a0c10a79b074fe --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_4_instances.txt @@ -0,0 +1,23 @@ +mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45 +mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529 +mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d +mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be +mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb +mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d +mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d +mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d +mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a +mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328 +mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52 +mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6 +mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5 +mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b +mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167 +mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632 +mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c +mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2 +mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53 +mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80 +mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183 +mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129 +mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7 diff --git a/0219194937_cosmos_minibench/gpu_5.log b/0219194937_cosmos_minibench/gpu_5.log new file mode 100644 index 0000000000000000000000000000000000000000..88d79aae67b0135aa83d56e4e064cb28b8f49f72 --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_5.log @@ -0,0 +1,46 @@ +[lock] Claimed: mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d +[lock] Claimed: mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5 +[lock] Claimed: mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e +[lock] Claimed: mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1 +[lock] Claimed: mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70 +[lock] Claimed: mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d +[lock] Claimed: mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d +[lock] Claimed: mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931 +[lock] Claimed: mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39 +[lock] Claimed: mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d +[lock] Claimed: mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc +[lock] Claimed: mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e +[lock] Claimed: mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290 +[lock] Claimed: mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2 +[lock] Claimed: mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b +[lock] Claimed: mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d +[lock] Claimed: mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea +[lock] Claimed: mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a +[lock] Claimed: mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8 +[lock] Claimed: mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c +[lock] Claimed: mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584 +[lock] Claimed: mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188 +[lock] Claimed: mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841 (worker=bravo-tongclass-a800:2659279) +[lock] Done: mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841 diff --git a/0219194937_cosmos_minibench/gpu_5_instances.txt b/0219194937_cosmos_minibench/gpu_5_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..cab720b2bc918808ff2c9784688879648dbb31c9 --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_5_instances.txt @@ -0,0 +1,23 @@ +mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d +mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5 +mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e +mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1 +mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70 +mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d +mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d +mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931 +mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39 +mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d +mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc +mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e +mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290 +mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2 +mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b +mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d +mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea +mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a +mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8 +mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c +mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584 +mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188 +mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841 diff --git a/0219194937_cosmos_minibench/gpu_6.log b/0219194937_cosmos_minibench/gpu_6.log new file mode 100644 index 0000000000000000000000000000000000000000..2bc49a0256f11f549ef8136ea3360354230c799e --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_6.log @@ -0,0 +1,44 @@ +[lock] Claimed: mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175 (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175 +[lock] Claimed: mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f +[lock] Claimed: mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a +[lock] Claimed: mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06 (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06 +[lock] Claimed: mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec +[lock] Claimed: mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5 (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5 +[lock] Claimed: mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a +[lock] Claimed: mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009 (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009 +[lock] Claimed: mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212 (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212 +[lock] Claimed: mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c +[lock] Claimed: mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b +[lock] Claimed: mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a +[lock] Claimed: mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819 (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819 +[lock] Claimed: mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c +[lock] Claimed: mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee +[lock] Claimed: mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f +[lock] Claimed: mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847 (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847 +[lock] Claimed: mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e +[lock] Claimed: mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f +[lock] Claimed: mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646 (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646 +[lock] Claimed: mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4 (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4 +[lock] Claimed: mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd (worker=bravo-tongclass-a800:2659277) +[lock] Done: mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd diff --git a/0219194937_cosmos_minibench/gpu_6_instances.txt b/0219194937_cosmos_minibench/gpu_6_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..2e7748d45c25c5adab62a124c20c31e7015d9077 --- /dev/null +++ b/0219194937_cosmos_minibench/gpu_6_instances.txt @@ -0,0 +1,22 @@ +mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175 +mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f +mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a +mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06 +mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec +mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5 +mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a +mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009 +mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212 +mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c +mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b +mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a +mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819 +mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c +mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee +mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f +mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847 +mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e +mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f +mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646 +mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4 +mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd diff --git a/0219194937_cosmos_minibench/mapping.json b/0219194937_cosmos_minibench/mapping.json new file mode 100644 index 0000000000000000000000000000000000000000..eb9878dc49c4e751660bca40f67c09a0fd10038d --- /dev/null +++ b/0219194937_cosmos_minibench/mapping.json @@ -0,0 +1,4538 @@ +[ + { + "instance_id": "mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ] + } + }, + { + "instance_id": "mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ], + "prompt_6": [ + 385, + 461 + ], + "prompt_7": [ + 462, + 538 + ], + "prompt_8": [ + 539, + 615 + ] + } + }, + { + "instance_id": "mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + } +] \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/config.yaml b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f700f44506469ad20007b9621ec649f861b4e6c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/console.log b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b3a9c78b321e23ea64c0fea03f547df991bf1ae6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/console.log @@ -0,0 +1,22 @@ +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/config.yaml +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.json +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:15:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:15:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:15:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:15:42|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/debug.log b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..23a094babb68cf4faeae15768e533f848c5abf2e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/debug.log @@ -0,0 +1,352 @@ +[02-20 16:01:28|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 16:01:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:01:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 16:01:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 16:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 16:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 16:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 16:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 16:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 16:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 16:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 16:01:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 16:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 16:01:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 16:01:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 16:01:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 16:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 16:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 16:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 16:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 16:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 16:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 16:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 16:01:41|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 16:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 16:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 16:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:01:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 16:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:01:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 16:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.61 s +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 16:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 16:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.27 s +[02-20 16:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:02:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:02:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/config.yaml +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 16:02:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a front view of the red bicycle, focusing on the 'STITCH' branding on the frame, the black seat, and the red handlebars. The red front basket with the 'HERO' logo is prominently displayed. As the camera rotates 72 degrees, the side profile becomes visible, revealing the silver rear rack and black tires with spokes. Continuing the rotation to 144 degrees, the red rear fender and the rear rack are shown from a different angle. At 216 degrees, the back of the bicycle is captured, emphasizing the rear rack and the red frame. Finally, rotating to 288 degrees, the side view reappears, and completing the 360-degree rotation, the video returns to the initial front view, ensuring the bicycle's identity is preserved throughout." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.json +[02-20 16:02:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:02:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:02:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 16:02:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 16:02:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 16:02:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 16:15:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 16:15:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:15:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:15:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:15:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 16:15:42|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..28be46cde8463515cc50b3e6c48939f9d28e3ef7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a front view of the red bicycle, focusing on the 'STITCH' branding on the frame, the black seat, and the red handlebars. The red front basket with the 'HERO' logo is prominently displayed. As the camera rotates 72 degrees, the side profile becomes visible, revealing the silver rear rack and black tires with spokes. Continuing the rotation to 144 degrees, the red rear fender and the rear rack are shown from a different angle. At 216 degrees, the back of the bicycle is captured, emphasizing the rear rack and the red frame. Finally, rotating to 288 degrees, the side view reappears, and completing the 360-degree rotation, the video returns to the initial front view, ensuring the bicycle's identity is preserved throughout.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1aa3eff0266d2b8d89db35176c44a6e757f7c3a6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1539efd7155d9ee964cf9252451765633b96db250cdacb2d3b342505eb2e29ad +size 1535084 diff --git a/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/config.yaml b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e9463257c83d0305644ca88cde5f935775391c7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/console.log b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/console.log new file mode 100644 index 0000000000000000000000000000000000000000..57c69181f138b337c5abb032a4249644d59bf674 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/console.log @@ -0,0 +1,22 @@ +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/config.yaml +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.json +[02-20 14:53:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:06:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:06:59|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/debug.log b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f985a039ba2c50b2c48f221e489a12d813bf2285 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/debug.log @@ -0,0 +1,352 @@ +[02-20 14:52:53|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 14:53:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:53:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:53:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 14:53:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 14:53:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 14:53:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 14:53:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 14:53:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 14:53:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 14:53:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 14:53:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 14:53:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 14:53:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 14:53:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 14:53:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 14:53:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 14:53:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 14:53:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 14:53:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 14:53:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 14:53:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 14:53:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 14:53:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 14:53:06|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 14:53:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 14:53:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:53:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:53:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 14:53:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:53:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 14:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:53:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 14:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 14:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:53:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:53:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:53:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:53:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 14:53:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 14:53:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.54 s +[02-20 14:53:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:53:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:53:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 14:53:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/config.yaml +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 14:53:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='Starting at 0°, the video presents a front view of the black bag with a red circular logo on the top and a red zipper pull on the side pocket, set against a patterned carpet. At 72°, the camera has rotated slightly to the right, bringing the black adjustable strap with the silver buckle into view on the side of the bag. By 144°, the rotation continues further to the right, offering a clear side profile of the bag, emphasizing the strap and buckle. At 216°, the camera has moved past the back of the bag, now showing the red zipper pull on the opposite side from the initial view. Finally, at 288°, the camera begins its return journey to the left, and by 360°, it completes the full 360-degree rotation, returning to the original front view of the bag with the red logo and zipper pull, ensuring a seamless loop.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.json +[02-20 14:53:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 14:53:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 15:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 15:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:06:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:06:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 15:06:59|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..11f31e495774bd47cc815ed1d0b84b830e487bbe --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Starting at 0°, the video presents a front view of the black bag with a red circular logo on the top and a red zipper pull on the side pocket, set against a patterned carpet. At 72°, the camera has rotated slightly to the right, bringing the black adjustable strap with the silver buckle into view on the side of the bag. By 144°, the rotation continues further to the right, offering a clear side profile of the bag, emphasizing the strap and buckle. At 216°, the camera has moved past the back of the bag, now showing the red zipper pull on the opposite side from the initial view. Finally, at 288°, the camera begins its return journey to the left, and by 360°, it completes the full 360-degree rotation, returning to the original front view of the bag with the red logo and zipper pull, ensuring a seamless loop.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bcb8e7c3615a5488c29674b9775623c5cb22df97 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5373daec9fd12b26ceb12ce411bc0b8609625a14652454672457067801ef123 +size 868229 diff --git a/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/config.yaml b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d4cbf461163c0020dfcd250551644ef79c299e6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/console.log b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..3ebecec75435f8d37a16bc01f3fed47d15c5c576 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/console.log @@ -0,0 +1,22 @@ +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:01:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:01:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/config.yaml +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.json +[02-20 04:01:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:15:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:15:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:15:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:15:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/debug.log b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bf2e57d0b2c3e920e8f7596df687976ad55328a4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/debug.log @@ -0,0 +1,352 @@ +[02-20 04:00:49|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 04:00:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:00:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:00:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 04:00:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 04:00:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 04:00:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 04:00:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 04:00:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 04:00:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 04:00:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 04:00:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 04:00:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 04:00:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 04:00:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 04:00:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 04:00:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 04:00:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 04:00:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 04:00:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 04:00:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 04:00:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 04:00:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 04:00:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 04:00:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 04:01:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 04:01:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 04:01:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 04:01:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 04:01:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 04:01:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 04:01:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 04:01:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 04:01:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 04:01:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 04:01:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 04:01:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:01:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:01:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 04:01:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:01:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:01:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:01:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 04:01:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:01:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 04:01:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:01:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:01:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 04:01:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:01:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 04:01:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 04:01:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 04:01:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 04:01:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:01:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 04:01:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 04:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 04:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.10 s +[02-20 04:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:01:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:01:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 04:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/config.yaml +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 04:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a direct front view of the carrot, focusing on its orange color, slight curve, and green leafy top. As the camera moves 72 degrees to the right, the side profile of the carrot becomes visible, showcasing its smooth texture and elongated shape. Continuing another 72 degrees, the camera reveals the back of the carrot, maintaining the same orange tone and subtle curvature. Another 72-degree rotation brings the carrot to the left side, offering a mirrored view of its smooth surface. Finally, the camera completes the 360-degree rotation by returning to the front view, ensuring the carrot's appearance remains consistent and recognizable throughout the entire sequence." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.json +[02-20 04:01:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 04:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 04:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 04:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 04:15:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 04:15:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:15:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:15:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:15:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 04:15:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..cc91db5031ba8d66c9a7678ada01a09ea1cc85c8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the carrot, focusing on its orange color, slight curve, and green leafy top. As the camera moves 72 degrees to the right, the side profile of the carrot becomes visible, showcasing its smooth texture and elongated shape. Continuing another 72 degrees, the camera reveals the back of the carrot, maintaining the same orange tone and subtle curvature. Another 72-degree rotation brings the carrot to the left side, offering a mirrored view of its smooth surface. Finally, the camera completes the 360-degree rotation by returning to the front view, ensuring the carrot's appearance remains consistent and recognizable throughout the entire sequence.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4b765c8d0da9b99c00b199d6347b8ccdfa14a55 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3406fb5ea2009d22e628d84c60def673a1ceb2be3c7a3291ded8ba1cdaa8383 +size 446811 diff --git a/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/config.yaml b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15107673864053b24c047150308f90636f5a1330 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/console.log b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ecc249d098404b791dad0158b00b38603edb28c1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/console.log @@ -0,0 +1,22 @@ +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:41:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:41:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/config.yaml +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.json +[02-19 23:42:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:55:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:55:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/debug.log b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a1d01810d44796e3bf28591be04293e0cbf878df --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/debug.log @@ -0,0 +1,352 @@ +[02-19 23:41:13|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 23:41:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:41:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 23:41:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 23:41:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 23:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 23:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 23:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 23:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 23:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 23:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 23:41:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 23:41:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 23:41:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 23:41:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 23:41:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 23:41:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 23:41:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 23:41:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 23:41:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 23:41:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 23:41:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 23:41:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 23:41:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 23:41:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 23:41:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 23:41:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 23:41:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 23:41:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 23:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:41:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 23:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:41:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 23:41:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:41:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-19 23:41:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 23:41:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 23:41:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 23:41:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:41:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 23:41:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 23:41:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 23:41:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:41:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 23:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 23:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.76 s +[02-19 23:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:42:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:42:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/config.yaml +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:42:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='Starting at 0 degrees, the video shows the front of the chocolate cake slice, with its dark brown base, white cream, and red filling clearly visible. The sliced almonds are arranged symmetrically around the cream. At 72 degrees, the camera shifts to a side view, highlighting the glossy surface and the texture of the chocolate and almonds. By 144 degrees, the view moves to the other side, showing the continuation of the almond arrangement and the depth of the cream layer. At 216 degrees, the camera captures the back of the slice, focusing on the overall layout of the almonds and the red filling. Finally, at 288 degrees, the view returns to the front, completing the 360-degree rotation and reinforcing the consistent identity of the cake slice.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.json +[02-19 23:42:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 23:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 23:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 23:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:55:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:55:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:55:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ea24e86cbf030fd9002dc34cd2110cf30cfea229 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Starting at 0 degrees, the video shows the front of the chocolate cake slice, with its dark brown base, white cream, and red filling clearly visible. The sliced almonds are arranged symmetrically around the cream. At 72 degrees, the camera shifts to a side view, highlighting the glossy surface and the texture of the chocolate and almonds. By 144 degrees, the view moves to the other side, showing the continuation of the almond arrangement and the depth of the cream layer. At 216 degrees, the camera captures the back of the slice, focusing on the overall layout of the almonds and the red filling. Finally, at 288 degrees, the view returns to the front, completing the 360-degree rotation and reinforcing the consistent identity of the cake slice.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38626929474e20619ec06d79fe980074ca5738b8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5090f0b294571b5580083d20f881beb2cfdbc4696641a5a27b826e38e33c5dd4 +size 892807 diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/config.yaml b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4a5ecbcb90886360dce834f112524874f7632fc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/console.log b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/console.log new file mode 100644 index 0000000000000000000000000000000000000000..f17cd9cbb9a868c5a59c71f8724368504e4f7759 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/console.log @@ -0,0 +1,22 @@ +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/config.yaml +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_6'] +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_6 +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.json +[02-20 04:53:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 05:07:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:07:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:07:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:07:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/debug.log b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f61374cf10d9a7f1f7397b512cb7ce4faafb8d51 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/debug.log @@ -0,0 +1,353 @@ +[02-20 04:53:11|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 04:53:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:53:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:53:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 04:53:20|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 04:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 04:53:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 04:53:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 04:53:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 04:53:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 04:53:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 04:53:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 04:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 04:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 04:53:20|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 04:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 04:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 04:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 04:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 04:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 04:53:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 04:53:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 04:53:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 04:53:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 04:53:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 04:53:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 04:53:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 04:53:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 04:53:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 04:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 04:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 04:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 04:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 04:53:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 04:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 04:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 04:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 04:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:53:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 04:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:53:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 04:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:53:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 04:53:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 04:53:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.32 s +[02-20 04:53:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:53:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:53:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 04:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/config.yaml +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_6'] +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_6 +[02-20 04:53:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_6' prompt_path=None prompt='Finally, the last 72-degree turn returns the scene to the original front view, completing the 360-degree rotation.' negative_prompt='' seed=48 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.json +[02-20 04:53:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.mp4 +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 04:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 04:54:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 04:54:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 05:07:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 05:07:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:07:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:07:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:07:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 05:07:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..4c5cd4360c4b2e4c7f8b031042a517f96ac38875 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a direct front view of two red apples on a green plastic stool.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9616dc88046c5e24ac76c0e37e87daacab593a91 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe04b7b9bfe8e66de48b7748fcac6e52ca605a5340a04dd17a6e98503ca6ad6c +size 544801 diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_1.json b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..907fbd934d649a249f945c99d7587ea46d09a776 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"The apples' glossy skin displays a gradient of red and yellow, with brown stems and small green leaves attached.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_1.mp4 b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3516dd3fb28c62d590a13dcd13523e9b4438bd0d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1595653e7372b1de9a3b0e4af8e31718a42cb72641b1c757cf1f041eaeecf4e0 +size 508409 diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_2.json b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a58d7d6e1e1a3ce95dd0950074798858258d3535 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The stool's woven top and cylindrical legs are distinct against the weathered wooden floor, which has colorful paint splatters.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_2.mp4 b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e5cd29b551ea5e7b3684d9d52a574a798244a643 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8999f6a70058eccdbd6f13585d94310e0bd5ed0488540dc94a90ba5a4b0e5f6 +size 1246795 diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_3.json b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a021f2a7d5cf45fd801140b7dda6fbc03053912a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the camera rotates 72 degrees to the right, the side view of the apples becomes visible, highlighting their roundness and the stems' position.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_3.mp4 b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23eba3376f324a06690d0dd212cab6dfcb942b5d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c149952168be5a32c78aec1609e88b8d8e5b00b3293909f0380287a2e0f45a52 +size 994662 diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_4.json b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..02df566ab266f90237d044618a35166a424e5f5c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Another 72-degree turn reveals the back of the stool and apples, showing the stems from this angle.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_4.mp4 b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cafb615e38486be9102328bfbba9eca9f6e8f494 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc91a03a5d4c536473d7deebc656a05bf2b82d1b34c69e224ba95b7f91563eed +size 861503 diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.json b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..d63c691166f5a7a8a79aa4d3c61a743c1396b6e6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"A further 72-degree rotation brings the apples closer to the front again, now viewed from a slightly different angle.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.mp4 b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..564fe9985f9ffdc63169f187da5a230a3539f806 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98ddef64692ed11085bb93fbc04274a418c2016ac4d17f89b3bc550c91731599 +size 899762 diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.json b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..478d614890043e5d7306c490cb71303397a07776 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Finally, the last 72-degree turn returns the scene to the original front view, completing the 360-degree rotation.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.mp4 b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2aa1bedc675e312cad4e45f3cf071bc97a3c5bab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900f5323f993644b8bf8b1130180199c07388acb4bb146750e9406557f5970c0 +size 1130544 diff --git a/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/config.yaml b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e82c28fe3b15a95fee8f3b9a86421db81e1f8d5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/console.log b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..a45f4b280689d5336a9f26444fb5aa9e9c0c9f46 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/console.log @@ -0,0 +1,22 @@ +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/config.yaml +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.json +[02-19 23:56:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:09:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:09:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:09:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:09:55|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/debug.log b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7ad0918dac34e91e2a3b97a751abde7ae6019ef1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/debug.log @@ -0,0 +1,352 @@ +[02-19 23:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 23:55:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:55:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 23:55:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 23:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 23:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 23:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 23:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 23:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 23:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 23:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 23:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 23:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 23:55:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 23:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 23:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 23:55:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 23:55:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 23:55:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 23:55:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 23:55:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 23:55:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 23:55:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 23:55:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 23:55:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 23:55:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 23:55:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 23:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 23:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 23:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 23:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 23:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 23:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 23:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 23:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 23:55:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 23:55:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 23:55:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:55:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:55:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 23:55:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 23:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:55:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 23:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.62 s +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 23:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:55:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 23:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 23:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.18 s +[02-19 23:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:56:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:56:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 23:56:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/config.yaml +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:56:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a direct front view of the ceramic bowl, emphasizing the central dark blue circle and the surrounding light blue stars. The camera begins its rotation, transitioning to a slightly angled side view that reveals the smooth curvature of the bowl and the extension of the leaf-like shapes along the inner edge. As the rotation continues, the viewer sees the bowl from a more pronounced side angle, showcasing the depth and texture of the design. Moving further, the back view is presented, displaying the mirrored pattern of the stars and leaf-like shapes. Finally, the camera completes its rotation, returning to the initial front view, ensuring the bowl's appearance remains consistent and recognizable throughout the entire sequence." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.json +[02-19 23:56:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 23:56:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 23:56:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 23:56:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 00:09:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 00:09:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:09:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:09:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:09:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 00:09:55|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..be4c496316cade73c2d4b138224c398f39edf1bc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a direct front view of the ceramic bowl, emphasizing the central dark blue circle and the surrounding light blue stars. The camera begins its rotation, transitioning to a slightly angled side view that reveals the smooth curvature of the bowl and the extension of the leaf-like shapes along the inner edge. As the rotation continues, the viewer sees the bowl from a more pronounced side angle, showcasing the depth and texture of the design. Moving further, the back view is presented, displaying the mirrored pattern of the stars and leaf-like shapes. Finally, the camera completes its rotation, returning to the initial front view, ensuring the bowl's appearance remains consistent and recognizable throughout the entire sequence.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..490d631d0226f2464153c4908c69436f23750736 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17377b8717c462edaad7e018f3963f5fc90b55a34d5961ba373b00f1042cba5c +size 848901 diff --git a/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/config.yaml b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36d28fda6eac60ad10a5af32535f99acd09bf0ff --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/console.log b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/console.log new file mode 100644 index 0000000000000000000000000000000000000000..78ac883c322cb5ff9a0ae406b7133a3feacce1d2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/console.log @@ -0,0 +1,22 @@ +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:30:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:30:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/config.yaml +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.json +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:44:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:44:23|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/debug.log b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..445cb428cd4eeb6f697a11a08e259917e33db94c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/debug.log @@ -0,0 +1,352 @@ +[02-20 07:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 07:30:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:30:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:30:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 07:30:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 07:30:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 07:30:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 07:30:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 07:30:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 07:30:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 07:30:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 07:30:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 07:30:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 07:30:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 07:30:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 07:30:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 07:30:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 07:30:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 07:30:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 07:30:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 07:30:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 07:30:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 07:30:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 07:30:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 07:30:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 07:30:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 07:30:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 07:30:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 07:30:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 07:30:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 07:30:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 07:30:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 07:30:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 07:30:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 07:30:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 07:30:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:30:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:30:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 07:30:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:30:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:30:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:30:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 07:30:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:30:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 07:30:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:30:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:30:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 07:30:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:30:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.04 s +[02-20 07:30:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 07:30:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 07:30:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 07:30:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:30:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:30:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:30:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 07:30:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 07:30:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.07 s +[02-20 07:30:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:31:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:31:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/config.yaml +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 07:31:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="Starting at 0°, the video shows a front view of the moss-like object, emphasizing its green color and fuzzy texture. At 72°, as the camera moves slightly to the side, the object's rounded shape and the light-colored base become more visible. By 144°, the object is viewed from a more pronounced side angle, highlighting its compact form and the subtle variations in its green shade. At 216°, the camera captures the back of the object, showing its rounded contours and the texture of its surface. Finally, from 288° to 360°, the camera returns to the front view, completing the rotation and ensuring the object's appearance remains consistent with the initial perspective." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.json +[02-20 07:31:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 07:31:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 07:31:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 07:31:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 07:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 07:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:44:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:44:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 07:44:23|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..6402217e8bbb475cd5ac776c29b4186a185c60f2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Starting at 0°, the video shows a front view of the moss-like object, emphasizing its green color and fuzzy texture. At 72°, as the camera moves slightly to the side, the object's rounded shape and the light-colored base become more visible. By 144°, the object is viewed from a more pronounced side angle, highlighting its compact form and the subtle variations in its green shade. At 216°, the camera captures the back of the object, showing its rounded contours and the texture of its surface. Finally, from 288° to 360°, the camera returns to the front view, completing the rotation and ensuring the object's appearance remains consistent with the initial perspective.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..994af532b6fe6bb84ec875a8c358eb3453bf38ce --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa9ef41458b0a89cccf9239861cde2f71ac3992a574a0a5eb65a237425cf24a +size 755054 diff --git a/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/config.yaml b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..daf9fa94f1e0a3836d47f1ee1843ba72b2584512 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/console.log b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/console.log new file mode 100644 index 0000000000000000000000000000000000000000..d15a0aebf74b1a5f32c888bbb5048632f79a442f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/console.log @@ -0,0 +1,22 @@ +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:48:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:48:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/config.yaml +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.json +[02-20 12:48:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:02:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:02:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:02:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:02:05|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/debug.log b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a672064470b5d5bf4d372c746835bd458a26a175 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/debug.log @@ -0,0 +1,352 @@ +[02-20 12:47:58|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 12:48:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:48:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 12:48:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 12:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 12:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 12:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 12:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 12:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 12:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 12:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 12:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 12:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 12:48:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 12:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 12:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 12:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 12:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 12:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 12:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 12:48:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 12:48:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 12:48:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 12:48:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 12:48:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 12:48:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 12:48:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 12:48:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 12:48:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 12:48:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 12:48:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 12:48:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 12:48:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 12:48:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 12:48:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:48:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:48:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 12:48:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:48:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:48:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:48:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 12:48:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:48:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 12:48:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:48:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:48:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 12:48:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:48:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 12:48:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 12:48:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.76 s +[02-20 12:48:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:48:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:48:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 12:48:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/config.yaml +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 12:48:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a direct front view of the small green crumpled object, which appears soft and irregular in shape, with a mix of dark and light green hues. As the camera rotates 72 degrees to the right, the object's side becomes visible, revealing its textured, crumpled surface and subtle depth. Continuing the rotation to 144 degrees, the object is now viewed from a slightly angled side perspective, emphasizing its irregular form and the way light reflects off its uneven surface. At 216 degrees, the camera shows the back of the object, highlighting its three-dimensional structure and the contrast between the darker and lighter green areas. Finally, as the camera rotates another 72 degrees to reach 288 degrees, the object is viewed from the opposite side, and as it completes the full 360-degree rotation, it returns to the initial front-facing view, maintaining consistency in appearance." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.json +[02-20 12:48:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 12:48:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 12:48:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 12:48:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 13:02:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 13:02:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:02:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:02:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:02:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 13:02:05|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ecdc9308c75cf25e5c9514efb70a1d4cb6809708 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a direct front view of the small green crumpled object, which appears soft and irregular in shape, with a mix of dark and light green hues. As the camera rotates 72 degrees to the right, the object's side becomes visible, revealing its textured, crumpled surface and subtle depth. Continuing the rotation to 144 degrees, the object is now viewed from a slightly angled side perspective, emphasizing its irregular form and the way light reflects off its uneven surface. At 216 degrees, the camera shows the back of the object, highlighting its three-dimensional structure and the contrast between the darker and lighter green areas. Finally, as the camera rotates another 72 degrees to reach 288 degrees, the object is viewed from the opposite side, and as it completes the full 360-degree rotation, it returns to the initial front-facing view, maintaining consistency in appearance.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48798895900d1b8ba6c2afa6d3f1ea0fd600568f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c240544c6438023029812de720bfdecf854a736af12805cc74848a2cf5779a01 +size 531782 diff --git a/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/config.yaml b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aa240ade1f65d201a4ae663a7ff8895373860e6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/console.log b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/console.log new file mode 100644 index 0000000000000000000000000000000000000000..2e7386b4a08c3490e8a3f113a944555d136e64f8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/console.log @@ -0,0 +1,22 @@ +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:22:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:22:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/config.yaml +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.json +[02-20 07:23:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:36:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:36:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:36:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:36:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/debug.log b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cd2280322de70465bd49066596fd56b905e0e693 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/debug.log @@ -0,0 +1,352 @@ +[02-20 07:22:33|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 07:22:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:22:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 07:22:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 07:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 07:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 07:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 07:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 07:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 07:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 07:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 07:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 07:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 07:22:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 07:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 07:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 07:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 07:22:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 07:22:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 07:22:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 07:22:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 07:22:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 07:22:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 07:22:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 07:22:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 07:22:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 07:22:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 07:22:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 07:22:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:22:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 07:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:22:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 07:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:22:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 07:22:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:22:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-20 07:22:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 07:22:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 07:22:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 07:22:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 07:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 07:22:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 07:22:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:22:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:23:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 07:23:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 07:23:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.51 s +[02-20 07:23:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:23:20|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:23:20|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 07:23:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/config.yaml +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 07:23:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video begins with a front-facing view of the black smartphone, focusing on the rectangular screen and the teal frame. The camera rotates 72 degrees to the right, revealing the side profile of the phone, which emphasizes its slim design and the teal border. Another 72-degree rotation brings the back of the phone into view, showcasing a flat surface. The camera continues to rotate another 72 degrees to the left, revealing the side profile again. Finally, the camera completes the 360-degree rotation by moving the last 72 degrees to the left, returning to the initial front view of the phone.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.json +[02-20 07:23:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 07:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 07:36:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 07:36:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:36:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:36:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:36:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 07:36:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ea659b4969bc031f05b99e40efc44f4b13a62cf7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a front-facing view of the black smartphone, focusing on the rectangular screen and the teal frame. The camera rotates 72 degrees to the right, revealing the side profile of the phone, which emphasizes its slim design and the teal border. Another 72-degree rotation brings the back of the phone into view, showcasing a flat surface. The camera continues to rotate another 72 degrees to the left, revealing the side profile again. Finally, the camera completes the 360-degree rotation by moving the last 72 degrees to the left, returning to the initial front view of the phone.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8db2c49f53ccb86b508a29b4fa50ba45fcdec052 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:428c28a2ac630e488ca018449d698dc92adb5cd84f4bd3ed2652b9e9ee927cbf +size 476117 diff --git a/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/config.yaml b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13d199f4eb7c4381575d7d07c37f750c8e08d026 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/console.log b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/console.log new file mode 100644 index 0000000000000000000000000000000000000000..3860103cd8315690f76fe82e0a58e0600627e659 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/console.log @@ -0,0 +1,22 @@ +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:10:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:10:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/config.yaml +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.json +[02-19 23:11:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:24:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:24:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:24:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:24:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/debug.log b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..88b0130cfa3484e232f208305522190f3c081296 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/debug.log @@ -0,0 +1,352 @@ +[02-19 23:10:21|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 23:10:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:10:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:10:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 23:10:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 23:10:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 23:10:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 23:10:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 23:10:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 23:10:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 23:10:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 23:10:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 23:10:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 23:10:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 23:10:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 23:10:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 23:10:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 23:10:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 23:10:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 23:10:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 23:10:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 23:10:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 23:10:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 23:10:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 23:10:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 23:10:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 23:10:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 23:10:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 23:10:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 23:10:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 23:10:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 23:10:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 23:10:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 23:10:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 23:10:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 23:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 23:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 23:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:10:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:10:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 23:10:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:10:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 23:10:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:10:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 23:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.95 s +[02-19 23:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 23:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 23:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 23:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 23:10:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:11:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 23:11:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 23:11:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.81 s +[02-19 23:11:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:11:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:11:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 23:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/config.yaml +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:11:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a direct front view of the decorative ceramic bowl, highlighting its white base, black and gray floral/geometric pattern, and the central orange egg yolk. The small red rectangular mark near the bottom right of the inner surface is subtly visible. As the camera rotates 72 degrees to the left, the side profile of the bowl comes into focus, emphasizing the texture and depth of the pattern. The egg yolk remains perfectly centered, and the red mark is still visible. Rotating another 72 degrees further, the camera captures the back of the bowl, where the pattern and the egg yolk's position remain consistent. Another 72-degree turn brings the camera to the right side of the bowl, maintaining the same visual details. Finally, as the camera completes the last 72-degree rotation, returning to the front view, the bowl appears identical to its initial state, ensuring a cohesive and continuous visual experience." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.json +[02-19 23:11:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 23:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 23:11:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 23:11:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:24:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:24:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:24:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:24:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:24:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:24:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2b6e4169c06e7e1a4e175b6adf23906ef0c39168 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a direct front view of the decorative ceramic bowl, highlighting its white base, black and gray floral/geometric pattern, and the central orange egg yolk. The small red rectangular mark near the bottom right of the inner surface is subtly visible. As the camera rotates 72 degrees to the left, the side profile of the bowl comes into focus, emphasizing the texture and depth of the pattern. The egg yolk remains perfectly centered, and the red mark is still visible. Rotating another 72 degrees further, the camera captures the back of the bowl, where the pattern and the egg yolk's position remain consistent. Another 72-degree turn brings the camera to the right side of the bowl, maintaining the same visual details. Finally, as the camera completes the last 72-degree rotation, returning to the front view, the bowl appears identical to its initial state, ensuring a cohesive and continuous visual experience.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..80295346f6e1d54cf245811990e2aa0a93b3e112 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:327c48150465951f6a50cb46303463947fd380bc36b004daa3b53cc4c033daae +size 1196871 diff --git a/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/config.yaml b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47dc716a6ede8b5641378cbc457ffb2a3060a812 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/console.log b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..f4669395dffb77f5348ceaddf097492314234796 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/console.log @@ -0,0 +1,22 @@ +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:58:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:58:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:58:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:58:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/config.yaml +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.json +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:12:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:12:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:12:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:12:13|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/debug.log b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b7dcfc6344d08994dfc8af07703ac2fdf492462d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/debug.log @@ -0,0 +1,352 @@ +[02-19 22:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 22:58:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:58:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:58:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 22:58:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 22:58:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 22:58:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 22:58:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 22:58:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 22:58:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 22:58:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 22:58:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 22:58:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 22:58:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 22:58:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 22:58:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 22:58:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 22:58:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 22:58:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 22:58:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 22:58:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 22:58:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 22:58:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 22:58:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 22:58:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 22:58:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 22:58:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 22:58:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 22:58:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 22:58:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:58:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:58:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:58:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 22:58:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:58:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 22:58:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:58:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 22:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-19 22:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 22:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 22:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 22:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 22:58:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 22:58:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 22:58:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:58:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:58:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 22:58:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 22:58:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.91 s +[02-19 22:58:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:58:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:58:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/config.yaml +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 22:58:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a front view of the duffel bag, focusing on its purple upper section, blue lower section, yellow zippers, and white text/logos, including '33' and partial letters like 'S' or 'E'. From 0-72°, the camera begins its smooth rotation to the right, gradually revealing the side profile of the bag. The side view highlights the bag's depth and the continuous yellow zippers running along the edge. Between 72-144°, the camera continues its movement, transitioning to a more direct side view that emphasizes the bag's compact, rectangular shape and fabric texture. From 144-216°, the camera moves further around the bag, now showing the back, which mirrors the front in terms of color scheme and zipper placement. Between 216-288°, the camera completes the rotation, bringing the bag closer to the front again but from a slightly different angle, showcasing the top and side details. Finally, from 288-360°, the camera returns to the original front view, ensuring the bag's identity is preserved and consistent throughout the entire 360-degree journey." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.json +[02-19 22:58:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 22:58:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 22:58:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 22:58:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:12:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:12:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:12:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:12:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:12:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:12:13|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..40d2b271ea938ddaad0b968a4c41f6827aa72d5f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a front view of the duffel bag, focusing on its purple upper section, blue lower section, yellow zippers, and white text/logos, including '33' and partial letters like 'S' or 'E'. From 0-72°, the camera begins its smooth rotation to the right, gradually revealing the side profile of the bag. The side view highlights the bag's depth and the continuous yellow zippers running along the edge. Between 72-144°, the camera continues its movement, transitioning to a more direct side view that emphasizes the bag's compact, rectangular shape and fabric texture. From 144-216°, the camera moves further around the bag, now showing the back, which mirrors the front in terms of color scheme and zipper placement. Between 216-288°, the camera completes the rotation, bringing the bag closer to the front again but from a slightly different angle, showcasing the top and side details. Finally, from 288-360°, the camera returns to the original front view, ensuring the bag's identity is preserved and consistent throughout the entire 360-degree journey.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05558f1e86256fb195a363a1e00df33d9dba6a8f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68c2612269f0598991ff46ac7d430852d0886dad6b00509ff34d7384a7209ebc +size 1581122 diff --git a/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/config.yaml b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..570aa4820485960540a6d08e3841d968e7b4f342 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/console.log b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..39db93cbe6cf8b42009ff92261f534055585c38b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/console.log @@ -0,0 +1,22 @@ +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/config.yaml +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.json +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:01:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:01:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/debug.log b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9a237286d6315e39ab7a61de699edaec2e329051 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/debug.log @@ -0,0 +1,352 @@ +[02-20 11:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 11:47:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:47:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:47:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 11:47:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 11:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 11:47:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 11:47:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 11:47:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 11:47:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 11:47:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 11:47:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 11:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 11:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 11:47:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 11:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 11:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 11:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 11:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 11:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 11:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 11:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 11:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 11:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 11:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 11:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 11:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 11:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 11:47:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 11:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 11:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:47:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 11:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 11:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 11:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 11:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 11:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 11:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.65 s +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.65 s +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 11:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:47:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:47:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 11:47:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 11:47:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.96 s +[02-20 11:47:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:48:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:48:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/config.yaml +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 11:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a direct front view of the peach-colored plastic chair, focusing on the blue 'ADR' logo on the upper backrest and the mesh-like pattern of the backrest. The armrests and seat are solid plastic, and the chair is positioned on a white tiled floor. As the camera rotates 72 degrees to the right (0-72°), the left side of the chair comes into view, showcasing the smooth, glossy texture of the plastic and the design of the armrests. Continuing the rotation to 144 degrees (72-144°), the chair's right side becomes visible, further emphasizing the uniformity of its color and texture. At 216 degrees (144-216°), the back of the chair is revealed, displaying the full mesh-like pattern and the even structure of the four legs. As the camera moves to 288 degrees (216-288°), the left side reappears, leading back toward the front. Finally, from 288 to 360 degrees (288-360°), the camera completes the 360-degree rotation, returning to the initial front view of the chair." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.json +[02-20 11:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 11:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 11:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 11:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 12:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 12:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:01:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:01:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 12:01:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..9b6499b7d8ae97018a3936d238e48748c7c5cf0a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the peach-colored plastic chair, focusing on the blue 'ADR' logo on the upper backrest and the mesh-like pattern of the backrest. The armrests and seat are solid plastic, and the chair is positioned on a white tiled floor. As the camera rotates 72 degrees to the right (0-72°), the left side of the chair comes into view, showcasing the smooth, glossy texture of the plastic and the design of the armrests. Continuing the rotation to 144 degrees (72-144°), the chair's right side becomes visible, further emphasizing the uniformity of its color and texture. At 216 degrees (144-216°), the back of the chair is revealed, displaying the full mesh-like pattern and the even structure of the four legs. As the camera moves to 288 degrees (216-288°), the left side reappears, leading back toward the front. Finally, from 288 to 360 degrees (288-360°), the camera completes the 360-degree rotation, returning to the initial front view of the chair.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..993eee8736cda07b800812921f20a6f1d01ede6f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d52725d69da51f1bbc6e728a1f49811b697db7ee31fd324d11c8fc4ab81ea64 +size 2493529 diff --git a/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/config.yaml b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62cce4fc09fbb0406d624e49427f5e5f96e08cb0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/console.log b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/console.log new file mode 100644 index 0000000000000000000000000000000000000000..d3d9fa5478cbe30a49d353287d3154750b427032 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/console.log @@ -0,0 +1,22 @@ +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/config.yaml +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.json +[02-20 13:42:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:55:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:55:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:55:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:55:42|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/debug.log b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a10e3517af626e91427f05520f7550626665f81e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/debug.log @@ -0,0 +1,352 @@ +[02-20 13:41:36|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 13:41:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:41:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:41:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 13:41:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 13:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 13:41:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 13:41:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 13:41:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 13:41:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 13:41:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 13:41:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 13:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 13:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 13:41:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 13:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 13:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 13:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 13:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 13:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 13:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 13:41:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 13:41:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 13:41:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 13:41:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 13:41:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 13:41:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 13:41:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 13:41:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 13:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 13:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 13:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 13:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 13:41:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 13:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 13:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 13:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 13:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:41:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 13:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 13:41:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:42:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 13:42:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 13:42:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.77 s +[02-20 13:42:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:42:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:42:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 13:42:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/config.yaml +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 13:42:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video starts with a direct front view of the white ceramic bowl, highlighting its scalloped edges and the small gray bird-like or floral patterns along the rim. The interior shows three small colored triangles near the center: yellow, red, and pink. As the camera rotates 72 degrees to the right (0-72°), the side profile of the bowl comes into view, revealing the smooth, glossy interior and the consistent placement of the colored triangles. Continuing the rotation (72-144°), the camera moves further around the bowl, showing more of the side and back, with the gray patterns along the rim remaining visible. At 144-216°, the back of the bowl is shown, maintaining the same decorative elements. As the camera rotates another 72 degrees (216-288°), the side profile reappears, and finally, from 288-360°, the camera completes the full 360-degree rotation, returning to the initial front view of the bowl.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.json +[02-20 13:42:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 13:42:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 13:42:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 13:42:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 13:55:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 13:55:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:55:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:55:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:55:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 13:55:42|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..371b830b774f6c042d040c8f9e371f07de7aa118 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the white ceramic bowl, highlighting its scalloped edges and the small gray bird-like or floral patterns along the rim. The interior shows three small colored triangles near the center: yellow, red, and pink. As the camera rotates 72 degrees to the right (0-72°), the side profile of the bowl comes into view, revealing the smooth, glossy interior and the consistent placement of the colored triangles. Continuing the rotation (72-144°), the camera moves further around the bowl, showing more of the side and back, with the gray patterns along the rim remaining visible. At 144-216°, the back of the bowl is shown, maintaining the same decorative elements. As the camera rotates another 72 degrees (216-288°), the side profile reappears, and finally, from 288-360°, the camera completes the full 360-degree rotation, returning to the initial front view of the bowl.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6b6cb7f06b1990adb2d5fb77d8725155fbd473d3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6941eac3919eaba22cae3944c207a9c6dbcc5f190720a9dac27f8babb6422ad +size 571684 diff --git a/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/config.yaml b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0209e92b4152fad23312f6182bbfa7e2d0c195eb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/console.log b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/console.log new file mode 100644 index 0000000000000000000000000000000000000000..0d35533ec8157f7a96f738d1ac2dd0866ebb8dcf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/console.log @@ -0,0 +1,22 @@ +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/config.yaml +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.json +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 09:03:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 09:03:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 09:03:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 09:03:15|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/debug.log b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d10abfc11c6d5205cc8238ed665d0ee60c1b54c2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/debug.log @@ -0,0 +1,352 @@ +[02-20 08:49:02|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 08:49:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:49:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:49:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 08:49:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 08:49:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 08:49:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 08:49:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 08:49:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 08:49:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 08:49:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 08:49:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 08:49:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 08:49:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 08:49:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 08:49:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 08:49:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 08:49:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 08:49:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 08:49:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 08:49:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 08:49:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 08:49:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 08:49:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 08:49:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 08:49:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 08:49:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 08:49:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 08:49:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 08:49:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 08:49:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 08:49:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 08:49:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 08:49:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 08:49:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 08:49:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:49:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:49:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 08:49:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:49:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 08:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:49:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 08:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 08:49:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:49:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:49:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 08:49:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 08:49:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.12 s +[02-20 08:49:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:49:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:49:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/config.yaml +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 08:49:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a front view of three orange carrots with green tops on a crumpled purple fabric. From 0-72°, the camera pans slightly to the right, offering a side perspective that showcases the carrots' smooth texture and subtle curves. The green leafy tops and natural root ends are still visible, maintaining the object's identity. Between 72-144°, the camera continues its rotation, moving further to the right and providing a more angled side view, emphasizing the carrots' elongated shapes. From 144-216°, the camera shifts to the back, displaying the full length of the carrots and their consistent orange color against the purple fabric. Between 216-288°, the camera moves to the left, transitioning from the back view to a front-facing angle similar to the initial shot. Finally, from 288-360°, the camera completes the 360-degree rotation by returning to the original front view, ensuring visual continuity and object consistency." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.json +[02-20 08:49:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 08:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 08:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 08:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 09:03:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 09:03:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 09:03:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 09:03:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 09:03:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 09:03:15|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..7c9ccd00509d9e3da8535f775c880b320707d7e6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a front view of three orange carrots with green tops on a crumpled purple fabric. From 0-72°, the camera pans slightly to the right, offering a side perspective that showcases the carrots' smooth texture and subtle curves. The green leafy tops and natural root ends are still visible, maintaining the object's identity. Between 72-144°, the camera continues its rotation, moving further to the right and providing a more angled side view, emphasizing the carrots' elongated shapes. From 144-216°, the camera shifts to the back, displaying the full length of the carrots and their consistent orange color against the purple fabric. Between 216-288°, the camera moves to the left, transitioning from the back view to a front-facing angle similar to the initial shot. Finally, from 288-360°, the camera completes the 360-degree rotation by returning to the original front view, ensuring visual continuity and object consistency.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8abd7875358c017929fa8a2dc639e886a6e7538f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac927d2bb683d276dd0e1496bef2f407e3833a230aee920a362c143f9c64ef44 +size 1405768 diff --git a/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/config.yaml b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d667d98e6e0aeef19ad03e8688a716ac5b655a55 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/console.log b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c0a3e36b0ecb1372478a0d61d963db01fc22169a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/console.log @@ -0,0 +1,22 @@ +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/config.yaml +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.json +[02-19 19:50:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 20:05:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 20:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 20:05:05|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/debug.log b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..16a778392af228a36922340f6e8ec28cd23a15af --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/debug.log @@ -0,0 +1,352 @@ +[02-19 19:49:47|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 19:49:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:49:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 19:49:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 19:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 19:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 19:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 19:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 19:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 19:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 19:49:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 19:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 19:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 19:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 19:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 19:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 19:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 19:49:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 19:49:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 19:49:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 19:49:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:50:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.15 s +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.67 s +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.67 s +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.67 s +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:50:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:50:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 19:50:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 19:50:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.49 s +[02-19 19:50:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:50:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 19:50:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/config.yaml +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 19:50:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a front view of the dark gray/black plastic chair, focusing on its perforated backrest and slightly curved design. As the camera rotates 72 degrees, the side profile of the chair becomes visible, showcasing its four straight legs and textured seat with minor scuffs. Continuing another 72 degrees, the back of the chair is revealed, emphasizing the vertical cutouts and perforated pattern. Rotating another 72 degrees, the camera captures the transition from an outdoor-like setting with blue barrels and greenery to an indoor environment with white walls and a wooden door. Finally, rotating the last 72 degrees, the camera returns to the initial front view, completing the 360-degree rotation and ensuring the chair's appearance remains consistent throughout." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.json +[02-19 19:50:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:50:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 19:50:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 19:50:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 19:50:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 20:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 20:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 20:05:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 20:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 20:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 20:05:05|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3f8a3971fb059abd8199048848715dcbf1b8a8fa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a front view of the dark gray/black plastic chair, focusing on its perforated backrest and slightly curved design. As the camera rotates 72 degrees, the side profile of the chair becomes visible, showcasing its four straight legs and textured seat with minor scuffs. Continuing another 72 degrees, the back of the chair is revealed, emphasizing the vertical cutouts and perforated pattern. Rotating another 72 degrees, the camera captures the transition from an outdoor-like setting with blue barrels and greenery to an indoor environment with white walls and a wooden door. Finally, rotating the last 72 degrees, the camera returns to the initial front view, completing the 360-degree rotation and ensuring the chair's appearance remains consistent throughout.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..afeb593c1a74e96de4ded01e2ccecd3c2a8327d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:800d080f65ef512498cb603bbe56dfbd35b745d4e72957039a166454d79edaf0 +size 1435766 diff --git a/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/config.yaml b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2960a79a0f2f51158d12076cda7372c979ab4faf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/console.log b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b7baa47bde4558671f89fb00d20e65dfefeaaeb3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/console.log @@ -0,0 +1,22 @@ +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:56:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:56:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/config.yaml +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.json +[02-19 22:56:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:10:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:10:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:10:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:10:11|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/debug.log b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..56b3eb31fab36c2fb350dd297af9e9a4707a9292 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/debug.log @@ -0,0 +1,352 @@ +[02-19 22:56:03|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 22:56:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:56:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 22:56:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 22:56:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 22:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 22:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 22:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 22:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 22:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 22:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 22:56:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 22:56:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 22:56:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 22:56:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 22:56:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 22:56:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 22:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 22:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 22:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 22:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 22:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 22:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 22:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 22:56:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 22:56:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 22:56:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 22:56:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 22:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:56:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:56:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 22:56:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:56:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 22:56:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:56:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 22:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 0.95 s +[02-19 22:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 22:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 22:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 22:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 22:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 22:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.67 s +[02-19 22:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:56:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 22:56:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/config.yaml +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 22:56:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="At the start, the video shows the front of the black rectangular box, highlighting the white text, yellow 'P' letters in the top left and bottom right corners, and three small headshot images near the bottom center. The logos for 'CPB' and 'F59', along with a barcode sticker, are clearly visible. As the camera moves 72 degrees to the left, the left side of the box becomes visible, showing more of the white text and the continuation of the structured layout. Continuing the rotation another 72 degrees (144 degrees total), the back of the box is revealed, displaying additional white text and the same yellow 'P' letters. Another 72-degree turn (216 degrees total) brings the right side of the box into view, which mirrors the front with all key features intact. Finally, as the camera completes the last 72-degree rotation (360 degrees total), it returns to the initial front view, ensuring the object's appearance remains consistent throughout the entire 360-degree journey." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.json +[02-19 22:56:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 22:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:10:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:10:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:10:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:10:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:10:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:10:11|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ec523e60377e4b03855af1fd0c043c9c6e37988a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"At the start, the video shows the front of the black rectangular box, highlighting the white text, yellow 'P' letters in the top left and bottom right corners, and three small headshot images near the bottom center. The logos for 'CPB' and 'F59', along with a barcode sticker, are clearly visible. As the camera moves 72 degrees to the left, the left side of the box becomes visible, showing more of the white text and the continuation of the structured layout. Continuing the rotation another 72 degrees (144 degrees total), the back of the box is revealed, displaying additional white text and the same yellow 'P' letters. Another 72-degree turn (216 degrees total) brings the right side of the box into view, which mirrors the front with all key features intact. Finally, as the camera completes the last 72-degree rotation (360 degrees total), it returns to the initial front view, ensuring the object's appearance remains consistent throughout the entire 360-degree journey.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b25bbe9290761ea0ba0a7ba306536db4990c358 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2f6a28c5e1306a3d36a0f45d5eeb6c8901bb64812768b34795ad33839417db +size 851825 diff --git a/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/config.yaml b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d54e07567221c02d36bf9667746775bfc3a0d9f2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/console.log b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/console.log new file mode 100644 index 0000000000000000000000000000000000000000..102fa765d8cf3f282a1f82ee2c047ca129e73019 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/console.log @@ -0,0 +1,22 @@ +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/config.yaml +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.json +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:55:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:55:06|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/debug.log b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..813c169b7a2a9ec7eb333412fc47b76cf890fdfb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/debug.log @@ -0,0 +1,352 @@ +[02-20 03:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 03:41:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:41:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:41:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 03:41:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 03:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 03:41:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 03:41:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 03:41:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 03:41:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 03:41:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 03:41:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 03:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 03:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 03:41:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 03:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 03:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 03:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 03:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 03:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 03:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 03:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 03:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 03:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 03:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 03:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 03:41:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 03:41:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 03:41:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 03:41:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 03:41:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 03:41:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 03:41:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 03:41:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 03:41:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 03:41:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:41:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:41:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 03:41:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 03:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:41:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 03:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 03:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:41:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:41:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 03:41:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 03:41:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.93 s +[02-20 03:41:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:41:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:41:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/config.yaml +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 03:41:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a front view of the book 'WASTE SPACE' by GINA DAMICO on a wooden floor. The black cover displays white text and a small spaceship graphic. As the camera rotates 72 degrees to the right, the white spine with black text becomes visible, showing the title and author's name. Rotating another 72 degrees (144 degrees total), the back of the book is revealed, maintaining the same black cover design but from a different perspective. Another 72-degree turn (216 degrees total) shows the left side of the book, including the other part of the spine and the front cover. A final 72-degree rotation (288 degrees total) brings the camera to the right side of the book, completing the 360-degree loop. The video concludes by returning to the initial front view of the book, ensuring all visual features remain consistent." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.json +[02-20 03:41:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 03:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 03:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 03:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 03:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 03:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:55:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:55:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 03:55:06|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..bfc7c108368129907b1146d9aa4ab120f67f6b49 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a front view of the book 'WASTE SPACE' by GINA DAMICO on a wooden floor. The black cover displays white text and a small spaceship graphic. As the camera rotates 72 degrees to the right, the white spine with black text becomes visible, showing the title and author's name. Rotating another 72 degrees (144 degrees total), the back of the book is revealed, maintaining the same black cover design but from a different perspective. Another 72-degree turn (216 degrees total) shows the left side of the book, including the other part of the spine and the front cover. A final 72-degree rotation (288 degrees total) brings the camera to the right side of the book, completing the 360-degree loop. The video concludes by returning to the initial front view of the book, ensuring all visual features remain consistent.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f769c75d203402f7f09a512b601d92f9a58e1def --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:183c6905dbe9769324e3b3fb25ac1bd98da860d0bfb9ad6aa0feded70a775c42 +size 798234 diff --git a/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/config.yaml b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcefcf6eef95d674bc820eb2da2542235669a3d4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/console.log b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..a7361a65a7baea1fd88a87d1d6e2622a26932f2a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/console.log @@ -0,0 +1,22 @@ +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:04:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:04:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/config.yaml +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.json +[02-19 20:04:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:18:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 20:18:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 20:18:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 20:18:19|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/debug.log b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..37138fbf308f7b621924991b9308694a426f6ac0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/debug.log @@ -0,0 +1,352 @@ +[02-19 20:04:10|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 20:04:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:04:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:04:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 20:04:20|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 20:04:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 20:04:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 20:04:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 20:04:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 20:04:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 20:04:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 20:04:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 20:04:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 20:04:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 20:04:20|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 20:04:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 20:04:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 20:04:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 20:04:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 20:04:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 20:04:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 20:04:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 20:04:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 20:04:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 20:04:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 20:04:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 20:04:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 20:04:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 20:04:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 20:04:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 20:04:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 20:04:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 20:04:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 20:04:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 20:04:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 20:04:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:04:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:04:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 20:04:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:04:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:04:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:04:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 20:04:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:04:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 20:04:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:04:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.02 s +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 20:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.15 s +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.68 s +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.68 s +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.68 s +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:04:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 20:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 20:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.31 s +[02-19 20:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:04:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:04:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 20:04:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/config.yaml +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 20:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video starts with a direct front view of the red plastic chair, focusing on the woven backrest pattern and the glossy finish of the seat. As the camera rotates 72 degrees to the right, the side profile of the chair becomes visible, emphasizing the armrests and the slight texture on the seat. Continuing the rotation another 72 degrees, the back of the chair is now in view, displaying the consistent woven pattern and the uniform red color. Another 72-degree turn brings the left side of the chair into focus, highlighting the armrests and the overall structure. Finally, as the camera completes the last 72-degree rotation, the front view is once again shown, mirroring the initial perspective and completing the 360-degree loop.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.json +[02-19 20:04:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 20:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 20:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 20:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 20:18:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 20:18:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 20:18:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 20:18:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 20:18:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 20:18:19|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea4c4585c1122d718a85134cb3b56b8194961c7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the red plastic chair, focusing on the woven backrest pattern and the glossy finish of the seat. As the camera rotates 72 degrees to the right, the side profile of the chair becomes visible, emphasizing the armrests and the slight texture on the seat. Continuing the rotation another 72 degrees, the back of the chair is now in view, displaying the consistent woven pattern and the uniform red color. Another 72-degree turn brings the left side of the chair into focus, highlighting the armrests and the overall structure. Finally, as the camera completes the last 72-degree rotation, the front view is once again shown, mirroring the initial perspective and completing the 360-degree loop.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..20c5e6fa1f2627afa7a484dbf9b437e91ea111e8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:409d0207c2e6fb7e8b1e9446676346e7212caa4e84a00df75d633afe066af82b +size 935335 diff --git a/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/config.yaml b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..965a72fd223e68ee7ea9f12f80351163dd6831ff --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/console.log b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/console.log new file mode 100644 index 0000000000000000000000000000000000000000..cab596069dcaf6b23f3e5e7f9f11f8e395ff3757 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/console.log @@ -0,0 +1,22 @@ +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:22:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:22:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/config.yaml +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.json +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:36:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:36:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:36:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:36:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/debug.log b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7fca41b26efcfc4da4b9466a4e634ef67ffaf5ac --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/debug.log @@ -0,0 +1,352 @@ +[02-20 06:22:39|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 06:22:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:22:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 06:22:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 06:22:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 06:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 06:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 06:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 06:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 06:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 06:22:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 06:22:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 06:22:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 06:22:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 06:22:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 06:22:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 06:22:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 06:22:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 06:22:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 06:22:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 06:22:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 06:22:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 06:22:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 06:22:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 06:22:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 06:22:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:22:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:22:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 06:22:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:22:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:22:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 06:22:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:22:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 06:22:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:22:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:22:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 06:22:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:22:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:22:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:22:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 06:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 06:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.37 s +[02-20 06:23:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:23:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 06:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:23:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/config.yaml +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 06:23:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='At the start of the video, the front of the bottle is visible, highlighting the black label with white pixelated squares and the golden-yellow liquid within. As the camera moves 72 degrees to the right, the side of the bottle becomes prominent, showing the same label design and liquid color. Continuing the rotation another 72 degrees (144 degrees total), the back of the bottle is now in view, maintaining the identical label pattern and liquid hue. Another 72-degree turn (216 degrees) brings the left side into focus, still displaying the consistent label and liquid. Finally, rotating the last 72 degrees (288-360 degrees), the front view reappears, completing the seamless 360-degree rotation.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.json +[02-20 06:23:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 06:23:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 06:23:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 06:23:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 06:36:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 06:36:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:36:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:36:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:36:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 06:36:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..4efbe1c2e6f2e296961ab76646f7a74a56820c62 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"At the start of the video, the front of the bottle is visible, highlighting the black label with white pixelated squares and the golden-yellow liquid within. As the camera moves 72 degrees to the right, the side of the bottle becomes prominent, showing the same label design and liquid color. Continuing the rotation another 72 degrees (144 degrees total), the back of the bottle is now in view, maintaining the identical label pattern and liquid hue. Another 72-degree turn (216 degrees) brings the left side into focus, still displaying the consistent label and liquid. Finally, rotating the last 72 degrees (288-360 degrees), the front view reappears, completing the seamless 360-degree rotation.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8cdb1bef722c5357d59b3e9a147cffc62995711c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e25c54a016f3a3857158768050ae7f1d42a63c074280a0e1a2413c2868fc15fc +size 1176314 diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/config.yaml b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d8437a74a4724758fa102606d7151db25fc3bc4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/console.log b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/console.log new file mode 100644 index 0000000000000000000000000000000000000000..6a206bade00a27190ccfa02c8ac89551532a266e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/console.log @@ -0,0 +1,22 @@ +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:33:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:33:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/config.yaml +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.json +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:47:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:47:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:47:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:47:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/debug.log b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9777e29b9a8b76cd0fbe84f9dd1e345408364985 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/debug.log @@ -0,0 +1,353 @@ +[02-20 12:33:37|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 12:33:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:33:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:33:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 12:33:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 12:33:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 12:33:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 12:33:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 12:33:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 12:33:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 12:33:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 12:33:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 12:33:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 12:33:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 12:33:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 12:33:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 12:33:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 12:33:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 12:33:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 12:33:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 12:33:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 12:33:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 12:33:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 12:33:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 12:33:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 12:33:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 12:33:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 12:33:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 12:33:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 12:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 12:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 12:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 12:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 12:33:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 12:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 12:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 12:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:33:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:33:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 12:33:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:33:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 12:33:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:33:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:33:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 12:33:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.65 s +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.66 s +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.66 s +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:33:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:34:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 12:34:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 12:34:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.17 s +[02-20 12:34:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:34:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:34:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/config.yaml +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 12:34:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final 288-360° segment, the camera completes its 360-degree rotation around the Schwinn mountain bike, returning to the initial front three-quarter view. The silver frame with blue and orange accents, black seat with blue trim, and black handlebars with green grips are once again the focal points. The orange reflectors on the wheels are visible, and the Schwinn logo on the down tube is clearly seen. The video concludes by maintaining the consistency of all visual features, ensuring a seamless loop back to the starting point.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.json +[02-20 12:34:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:34:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.mp4 +[02-20 12:34:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 12:34:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 12:34:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 12:34:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 12:47:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 12:47:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:47:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:47:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:47:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 12:47:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ecf7da0c06d4ebf03c7f6a3f94b624336f61f06c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a front three-quarter view of the Schwinn mountain bike, showcasing its silver frame with blue and orange accents, black seat with blue trim, and black handlebars with green grips. The orange reflectors on the wheels are prominently displayed. The camera begins its rotation to the left, capturing the side profile of the bike, including the suspension forks, rear derailleur, and chainstays. The Schwinn logo on the down tube is visible, and the bike's components are highlighted from this angle.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a0d6168e1b5b959af1eff1a77eab770d3d0eeedf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71eaf0e4893339fbe13106cf9ac65baa4d65aa9277dc05483c4487a22b6260ff +size 1018688 diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_1.json b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..67e8d3c22c0b707122faa982a76278c84a35cf76 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its rotation (72-144°), it moves past the side profile of the Schwinn mountain bike, focusing on the rear wheel, fender, and seat post. The orange reflectors on the wheels remain visible, and the Schwinn logo on the down tube is still in view. The black seat with blue trim and the black handlebars with green grips are seen from different angles, providing a detailed look at the bike's features. The camera then starts to move back towards the front, preparing for the next segment.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_1.mp4 b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..987048aeab8018a289e419c095ad09bba3f9d6b3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f184ab503fb1d38cc12c402fa663e7906a95eac2afbd9774555f1e8e37581315 +size 980136 diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_2.json b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e0441b15c4221f5d0c62a302bb76aee104a9f592 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the 144-216° segment, the camera captures the rear view of the Schwinn mountain bike, emphasizing the rear wheel, fender, and seat post. The orange reflectors on the wheels are still noticeable, and the Schwinn logo on the down tube is visible. The black seat with blue trim and the black handlebars with green grips are seen from various angles, highlighting the bike's components. The camera continues to rotate, bringing the bike closer to the front view.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_2.mp4 b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57164e108362c18faffa5bc9a252ad8551183292 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55fe36cb9f852e0cc8332e35f80b5ca2b27eba9c55bc82832d15b403da92072a +size 964911 diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.json b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..18ccae423846c538887518295f150b74c017cdb8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The 216-288° segment focuses on the side and front views of the Schwinn mountain bike. The silver frame with blue and orange accents, black seat with blue trim, and black handlebars with green grips are prominently displayed. The orange reflectors on the wheels are visible, and the Schwinn logo on the down tube is clearly seen. The camera captures the bike's components from different angles, ensuring all visual features remain consistent. The rotation continues, bringing the bike back towards the front.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.mp4 b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7689565b7fa7d69199bf0d691d85f86e8e9a68af --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05a39bc5c8fa687305d0fe0274758e7d93653f8b5f6dcac7d062fbc9b42c0ced +size 1022261 diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.json b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..0eb6c47b53b2eb8f0bea135b63bf4ade3b97eff0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final 288-360° segment, the camera completes its 360-degree rotation around the Schwinn mountain bike, returning to the initial front three-quarter view. The silver frame with blue and orange accents, black seat with blue trim, and black handlebars with green grips are once again the focal points. The orange reflectors on the wheels are visible, and the Schwinn logo on the down tube is clearly seen. The video concludes by maintaining the consistency of all visual features, ensuring a seamless loop back to the starting point.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.mp4 b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e3ba66dc79c725adbaf6a77c90554d02d8a146b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c6944f60de0dac3bda8ded2f024007bfed9a8c738d1be09372728ce7f0808ed +size 1195560 diff --git a/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/config.yaml b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c0bf300d37bde0c8c6563f030cd9c83ed1ee21e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/console.log b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/console.log new file mode 100644 index 0000000000000000000000000000000000000000..be2f3823d40d26e31504c9adcc47d904f3e0f20e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/console.log @@ -0,0 +1,22 @@ +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:21:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:21:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/config.yaml +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.json +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:35:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:35:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:35:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:35:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/debug.log b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..59f3c3b81ddb14a59b4478131899bbfeb4d6bffb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/debug.log @@ -0,0 +1,352 @@ +[02-20 00:21:12|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 00:21:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:21:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:21:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 00:21:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 00:21:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 00:21:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 00:21:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 00:21:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 00:21:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 00:21:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 00:21:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 00:21:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 00:21:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 00:21:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 00:21:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 00:21:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 00:21:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 00:21:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 00:21:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 00:21:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 00:21:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 00:21:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 00:21:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 00:21:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 00:21:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 00:21:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 00:21:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 00:21:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 00:21:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 00:21:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 00:21:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 00:21:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 00:21:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 00:21:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 00:21:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 00:21:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:21:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:21:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 00:21:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:21:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:21:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:21:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 00:21:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:21:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 00:21:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:21:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:21:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 00:21:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:21:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 00:21:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 00:21:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 00:21:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 00:21:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:21:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:21:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 00:21:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 00:21:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.61 s +[02-20 00:21:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:22:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:22:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/config.yaml +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 00:22:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='At the start of the video, the broccoli floret is positioned centrally on the countertop, with the front view highlighting its bright green floret and dual stems. The camera begins its rotation, capturing the left side of the floret in the first 72 degrees, where the texture of the buds and the thickness of the stems are more pronounced. In the next 72 degrees (72-144°), the side profile becomes dominant, showing the elongated shape of the stems and the compactness of the floret. From 144-216°, the back of the floret is revealed, with the dense arrangement of buds becoming the focal point. As the rotation progresses from 216-288°, the right side of the floret comes into view, mirroring the initial side profile but from the opposite angle. Finally, in the last 72 degrees (288-360°), the camera completes the 360-degree orbit, returning to the original front view of the broccoli floret.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.json +[02-20 00:22:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:22:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 00:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 00:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 00:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 00:35:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 00:35:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:35:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:35:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:35:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 00:35:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a0e72a838f5217abc7dd65af5eb53289928a66bb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"At the start of the video, the broccoli floret is positioned centrally on the countertop, with the front view highlighting its bright green floret and dual stems. The camera begins its rotation, capturing the left side of the floret in the first 72 degrees, where the texture of the buds and the thickness of the stems are more pronounced. In the next 72 degrees (72-144°), the side profile becomes dominant, showing the elongated shape of the stems and the compactness of the floret. From 144-216°, the back of the floret is revealed, with the dense arrangement of buds becoming the focal point. As the rotation progresses from 216-288°, the right side of the floret comes into view, mirroring the initial side profile but from the opposite angle. Finally, in the last 72 degrees (288-360°), the camera completes the 360-degree orbit, returning to the original front view of the broccoli floret.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dabb4c4a375c65c9fe0b7145dac54e77c1a60cac --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e2818eaab09157399252ecfd4eac30a78d31c81202c9c8186588aa7dcd4dc10 +size 1356992 diff --git a/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/config.yaml b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27c81727b6e7e5a6cb496d0427736f4be039b980 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/console.log b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/console.log new file mode 100644 index 0000000000000000000000000000000000000000..4094e200b8ddb1d1fb01ef23d5d541193becc772 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/console.log @@ -0,0 +1,22 @@ +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/config.yaml +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.json +[02-19 19:50:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:03:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 20:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 20:03:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 20:04:00|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/debug.log b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c77deca0f7b8ebd61a38777d2f241d8e7f3824d6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/debug.log @@ -0,0 +1,352 @@ +[02-19 19:49:47|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 19:49:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:49:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:49:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 19:49:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 19:49:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 19:49:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 19:49:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 19:49:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 19:49:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 19:49:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 19:49:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 19:49:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 19:49:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 19:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 19:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 19:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 19:49:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 19:49:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 19:49:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 19:49:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 19:49:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:49:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:50:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 19:50:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:50:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.15 s +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.70 s +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.71 s +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.71 s +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:50:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:50:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:50:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 19:50:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 19:50:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.91 s +[02-19 19:50:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:50:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:50:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 19:50:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/config.yaml +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 19:50:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a front view of the orange carrot on the checkered tablecloth. The carrot's bright orange body, smooth texture, green leafy top, and brown stem are clearly visible. The camera begins its rotation to the right (0-72°), gradually revealing the side profile of the carrot. The green leafy top is now seen from an angle, and the brown stem is partially visible. The background remains blurred, keeping the focus on the carrot. As the camera continues to rotate (72-144°), the side and back views of the carrot become more prominent, showcasing the cylindrical shape and the brown stem. The green leafy top is still visible but from a different perspective. The next segment (144-216°) further rotates the camera, providing a clearer view of the back of the carrot, where the brown stem is most noticeable. The green leafy top is now seen from behind. The following segment (216-288°) continues the rotation, bringing the carrot closer to the front view again, with the green leafy top becoming more visible from the side. Finally, the camera completes the 360-degree rotation (288-360°), returning to the initial front view of the carrot, ensuring a seamless loop." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.json +[02-19 19:50:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 19:50:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 19:50:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 19:50:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 20:03:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 20:03:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 20:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 20:03:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 20:03:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 20:04:00|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..4486c67387b18f360d5556a45706f7926f22590a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a front view of the orange carrot on the checkered tablecloth. The carrot's bright orange body, smooth texture, green leafy top, and brown stem are clearly visible. The camera begins its rotation to the right (0-72°), gradually revealing the side profile of the carrot. The green leafy top is now seen from an angle, and the brown stem is partially visible. The background remains blurred, keeping the focus on the carrot. As the camera continues to rotate (72-144°), the side and back views of the carrot become more prominent, showcasing the cylindrical shape and the brown stem. The green leafy top is still visible but from a different perspective. The next segment (144-216°) further rotates the camera, providing a clearer view of the back of the carrot, where the brown stem is most noticeable. The green leafy top is now seen from behind. The following segment (216-288°) continues the rotation, bringing the carrot closer to the front view again, with the green leafy top becoming more visible from the side. Finally, the camera completes the 360-degree rotation (288-360°), returning to the initial front view of the carrot, ensuring a seamless loop.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..040636c6ff5eeef05027afd64a00cae205b9c8e9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf641207d52cad313f94ee8e693312643bb179c3204c11366ac2c059ba71eb35 +size 953569 diff --git a/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/config.yaml b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38b30a3553f50791079424f50dde3e1f81580c93 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/console.log b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/console.log new file mode 100644 index 0000000000000000000000000000000000000000..5949ae0050e0469a484bd94bb08281999fe07f9a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/console.log @@ -0,0 +1,22 @@ +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:55:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:55:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/config.yaml +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.json +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 22:09:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 22:09:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 22:09:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 22:09:33|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/debug.log b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7c51804f763e7a45be668a01ac609ad57128d94e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/debug.log @@ -0,0 +1,352 @@ +[02-20 21:55:22|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 21:55:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:55:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 21:55:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 21:55:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 21:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 21:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 21:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 21:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 21:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 21:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 21:55:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 21:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 21:55:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 21:55:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 21:55:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 21:55:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 21:55:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 21:55:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 21:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 21:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 21:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 21:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 21:55:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 21:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 21:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 21:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:55:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:55:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:55:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 21:55:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:55:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 21:55:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:55:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:55:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:55:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:55:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 21:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 21:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.28 s +[02-20 21:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:56:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:56:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/config.yaml +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 21:56:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a direct front view of the book 'Along Came Spider' by James Patterson, positioned upright on a speckled granite table. The cover features a red and blue design with a brick wall image, and the title is prominently displayed in white and yellow font. As the camera rotates 72 degrees to the right, the side profile of the book comes into view, showcasing the spine with its blue and white coloration and the author's name. Continuing the rotation to 144 degrees, the back of the book becomes visible, revealing additional text and design elements. At 216 degrees, the camera has moved further around, showing the other side profile of the book. Finally, as the camera reaches 288 degrees, it begins to move back towards the front, completing the 360-degree rotation and returning to the initial front view of the book." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.json +[02-20 21:56:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:56:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 21:56:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 21:56:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 21:56:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 22:09:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 22:09:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 22:09:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 22:09:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 22:09:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 22:09:33|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..759c22862075b817871d48441a1f80dbcfae00b4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the book 'Along Came Spider' by James Patterson, positioned upright on a speckled granite table. The cover features a red and blue design with a brick wall image, and the title is prominently displayed in white and yellow font. As the camera rotates 72 degrees to the right, the side profile of the book comes into view, showcasing the spine with its blue and white coloration and the author's name. Continuing the rotation to 144 degrees, the back of the book becomes visible, revealing additional text and design elements. At 216 degrees, the camera has moved further around, showing the other side profile of the book. Finally, as the camera reaches 288 degrees, it begins to move back towards the front, completing the 360-degree rotation and returning to the initial front view of the book.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d7543faed2641de5dd2c0641325dbdb86bbbef --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a792ee2cf9310375d73eff9b378ca2d4200a5fffdc8cf4389c0f493ead84f8b +size 1022301 diff --git a/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/config.yaml b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29040ade823455611f801f6a581cd9e26b348c84 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/console.log b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/console.log new file mode 100644 index 0000000000000000000000000000000000000000..40a3c564bba22ad39d542ce8626e803d45724eae --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/console.log @@ -0,0 +1,22 @@ +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:33:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:33:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/config.yaml +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.json +[02-20 21:33:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:47:11|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/debug.log b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6cdfbdf3ad7d4f326bd32d26bd4b139266acc08a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/debug.log @@ -0,0 +1,352 @@ +[02-20 21:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 21:33:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:33:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 21:33:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 21:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 21:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 21:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 21:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 21:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 21:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 21:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 21:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 21:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 21:33:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 21:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 21:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 21:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 21:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 21:33:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 21:33:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 21:33:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 21:33:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 21:33:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 21:33:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 21:33:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 21:33:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 21:33:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 21:33:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 21:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 21:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 21:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 21:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 21:33:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 21:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 21:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 21:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:33:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:33:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 21:33:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:33:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 21:33:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:33:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:33:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 21:33:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:33:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 21:33:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 21:33:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 21:33:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 21:33:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:33:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 21:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 21:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.36 s +[02-20 21:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:33:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:33:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 21:33:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/config.yaml +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 21:33:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video starts with a direct front view of the wooden bench, focusing on the detailed animal silhouettes on the backrest and the peeling paint on the seat. The black metal armrests and frame are prominent. As the camera moves 72 degrees to the right, the side of the bench becomes visible, revealing the texture of the weathered wood and the elegant curves of the metal legs. Continuing the rotation, at 144 degrees, the bench is shown from a three-quarter angle, emphasizing the depth and craftsmanship of the design. At 216 degrees, the camera provides a rear view, highlighting the full extent of the animal silhouettes and the overall structure. Finally, as the camera reaches 288 degrees, the bench is viewed from the opposite side, mirroring the earlier side profile before completing the rotation back to the front at 360 degrees.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.json +[02-20 21:33:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 21:33:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 21:33:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 21:33:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 21:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 21:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:47:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 21:47:11|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..7abcba3484d1bc70ab58d758a58596cbbc2ff3b0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the wooden bench, focusing on the detailed animal silhouettes on the backrest and the peeling paint on the seat. The black metal armrests and frame are prominent. As the camera moves 72 degrees to the right, the side of the bench becomes visible, revealing the texture of the weathered wood and the elegant curves of the metal legs. Continuing the rotation, at 144 degrees, the bench is shown from a three-quarter angle, emphasizing the depth and craftsmanship of the design. At 216 degrees, the camera provides a rear view, highlighting the full extent of the animal silhouettes and the overall structure. Finally, as the camera reaches 288 degrees, the bench is viewed from the opposite side, mirroring the earlier side profile before completing the rotation back to the front at 360 degrees.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..83f9e043f9c5f350bd5203a8b62aa5b254a33d71 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9161ad596d9ccf2ea33579e68fc03a7c9f1bcac57b91c88d02f54092ad146c5c +size 954388 diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/config.yaml b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c39e2759fb3d21e9a8c2688f55dfe214b6e012e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/console.log b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/console.log new file mode 100644 index 0000000000000000000000000000000000000000..baf70e687bf07f1e0ea4853b1a77e97dfe87e3c6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/console.log @@ -0,0 +1,22 @@ +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:21:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:21:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/config.yaml +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.json +[02-20 17:22:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:36:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:36:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:36:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:36:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/debug.log b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6a0fe8c2c5f6d9063001e9b20d366fe2e5d2e225 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/debug.log @@ -0,0 +1,353 @@ +[02-20 17:21:42|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 17:21:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:21:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:21:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 17:21:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 17:21:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 17:21:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 17:21:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 17:21:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 17:21:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 17:21:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 17:21:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 17:21:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 17:21:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 17:21:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 17:21:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 17:21:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 17:21:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 17:21:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 17:21:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 17:21:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 17:21:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 17:21:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 17:21:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 17:21:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 17:21:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 17:21:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 17:21:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 17:21:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 17:21:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 17:21:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 17:21:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 17:21:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 17:21:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 17:21:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 17:21:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:21:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:21:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 17:21:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:21:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:21:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:21:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 17:21:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:21:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 17:21:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:21:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:21:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 17:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 17:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.86 s +[02-20 17:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:22:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:22:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/config.yaml +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 17:22:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, as the camera reaches 288 degrees and completes the full 360-degree rotation, it returns to the initial front view, ensuring the backpack's identity and visual consistency are maintained throughout the video." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.json +[02-20 17:22:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.mp4 +[02-20 17:22:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 17:22:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 17:22:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 17:22:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 17:36:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 17:36:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:36:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:36:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:36:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 17:36:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..76fdbb337200d4c0c3e06efdff82c38d10221e38 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a clear front view of the black backpack, emphasizing its textured fabric and the red and white logo on the upper front pocket.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e504d7d3483a6de525af1ebfc5fcced1e8e8764 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e6ceb688957a407c61e0e86f9f32ff5fd7cede5d7185cbf7c392ed9ea389aa2 +size 1703332 diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_1.json b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..3ebb4241b7175a703a328dc87ae9cfe01d84166a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera moves 72 degrees to the right, the side profile becomes visible, showcasing the backpack's side pockets and straps.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_1.mp4 b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e586d3514052ee182f349f616b6bc1fb3c65595d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92d41c2a94d3d73f6566446b0025422c78337262fdffd982d280bec2a3054247 +size 1561000 diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_2.json b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..64f1c602f42fd52ce72e7d8d30e1427d76d70d0d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Continuing the rotation, at 144 degrees, the back of the backpack is revealed, displaying additional zippers and structural elements.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_2.mp4 b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f476be2f5f473fd373b7ca9d5b7948e9e94885f0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c750e2c9622b5e871a12c1134fca0ffdea8fd4aa6ea2b460c6ae40dea087efd +size 1236389 diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.json b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..97fa43a7beda07f99cb8e5734967131fa241146b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"By 216 degrees, the camera has rotated further, providing a comprehensive view of the backpack's design from various angles.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.mp4 b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9161119bf55a09aebe724cab9d3bb3fd94b9506b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2424367f7b13cd50d48985e5ef4a0fd7aef46bfe4dfa5278c12e81a771e7e53 +size 1202083 diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.json b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ba503c361a2bcf4449d5977569a51f794399159d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, as the camera reaches 288 degrees and completes the full 360-degree rotation, it returns to the initial front view, ensuring the backpack's identity and visual consistency are maintained throughout the video.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.mp4 b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88789498742ebdbac27d46617f59e9f6227efb20 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b3d0a204e5666ce04dd15a0fb706cec74180bcdfc1c9ea4d852636b1e44ef17 +size 1130992 diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/config.yaml b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3318e8960e010a696454af7e86176d573194a0e6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/console.log b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ee9760b97f4c849de56ae5cf94d71433deab89c4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/console.log @@ -0,0 +1,22 @@ +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/config.yaml +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.json +[02-20 13:28:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:41:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:41:25|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/debug.log b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..196a31d5b04dea918b04ef62fdb652abc816b94d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/debug.log @@ -0,0 +1,353 @@ +[02-20 13:27:20|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 13:27:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 13:27:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 13:27:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 13:27:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 13:27:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 13:27:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 13:27:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 13:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 13:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 13:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 13:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 13:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 13:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 13:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 13:27:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 13:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 13:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 13:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 13:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:27:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 13:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 13:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.66 s +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.66 s +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.66 s +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:27:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:28:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 13:28:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 13:28:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.34 s +[02-20 13:28:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:28:06|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:28:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/config.yaml +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 13:28:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the camera completes the 360-degree rotation, returning to the initial front view of the bench. The vibrant green grassy surface and the consistent design of the bench are showcased, ensuring a seamless loop and confirming the bench's geometric integrity throughout the entire sequence." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.json +[02-20 13:28:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:28:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.mp4 +[02-20 13:28:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 13:28:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 13:28:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 13:28:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 13:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 13:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:41:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:41:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:41:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 13:41:25|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ad8a8fd23372994df23ee45e80688e8f7f40cf9f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a front view of the wooden park bench, highlighting its light gray wooden slats and black metal frame. The camera begins its rotation, moving to the right side of the bench, revealing the ornate armrests and the consistent spacing between the slats. The vibrant green grassy surface beneath the bench adds contrast to the scene.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa3d0688c6f39df2f035e6aa96e4523727659283 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaaad1f616a81a30740464de8838b14fbde48556bcc230908be81eab0fffd6f4 +size 1067215 diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_1.json b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..1ee94aeeb4577cf2a5b5f0c01b92311ccaaca367 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its rotation, it moves to the left side of the bench, providing a clear view of the armrests and the underside of the seat. The consistent spacing between the wooden slats and the black metal frame remains evident, emphasizing the bench's uniform design.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_1.mp4 b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..90cb2f5e460ce87e57bccb0c78158987adce46f4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87fac3674bcd2d82d733ccaa58b8cea6653772678cc115169d38033660622ad2 +size 1604013 diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_2.json b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..13396c4a60a1eeffcec282cef82db43298b207cf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The rotation progresses further, with the camera focusing on the backrest of the bench. The light gray wooden slats and black metal frame are consistently displayed, with no signs of distortion. The camera captures the bench from various angles, ensuring that the geometric integrity is maintained.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_2.mp4 b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eded2554c0d848d4f161a6b738b7ef71013a2f19 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ae83ea132232a26c0bd80a163308d1c5711725f00f3d9d1bc7f49e954f1ebfe +size 961019 diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.json b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..429fab4d5f8639b2a38295dedb5013cdfb352d66 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Moving to the right side of the bench, the camera provides a comprehensive view of the armrests and the overall structure. The consistent spacing between the wooden slats and the black metal frame is reinforced, highlighting the bench's stability and design.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.mp4 b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..279892ec50be8ac64e4ff7623cddaf144fcdb218 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:170a014429d656047e6332cdb37d6120bc2201c8be4aa9ca2ee04b015fee31b2 +size 713628 diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.json b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..801e9a8dfd8a2873cd76ded1d470d94a3114afb0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the camera completes the 360-degree rotation, returning to the initial front view of the bench. The vibrant green grassy surface and the consistent design of the bench are showcased, ensuring a seamless loop and confirming the bench's geometric integrity throughout the entire sequence.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.mp4 b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..43b5e67f7105e9d6f2621cffbfc2a5379af3876e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b066608f7b8813b2088f0d9862783cbba3d53f7ea573ecb3e0ee9f77af10e67 +size 1947048 diff --git a/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/config.yaml b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a21d515949a14e16e60aa98ef49f346db0934e0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/console.log b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/console.log new file mode 100644 index 0000000000000000000000000000000000000000..53289266df7c36269be7912a2af4222298f5bed6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/console.log @@ -0,0 +1,22 @@ +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:57:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:57:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/config.yaml +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.json +[02-20 14:57:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:10:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:10:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:10:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:10:56|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/debug.log b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8957975b784413aec5ec26b4939c353c15f255eb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/debug.log @@ -0,0 +1,352 @@ +[02-20 14:56:49|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 14:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:56:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 14:56:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 14:56:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 14:56:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 14:56:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 14:56:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 14:56:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 14:56:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 14:56:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 14:56:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 14:56:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 14:56:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 14:56:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 14:56:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 14:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 14:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 14:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 14:57:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 14:57:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 14:57:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 14:57:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 14:57:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 14:57:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 14:57:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:57:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:57:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 14:57:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:57:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:57:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 14:57:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:57:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 14:57:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:57:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:57:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 14:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 14:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.69 s +[02-20 14:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:57:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:57:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 14:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/config.yaml +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 14:57:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a direct front view of the black smartphone on the wooden table, emphasizing its rectangular shape and rounded corners. From 0-72°, the camera begins its rotation to the right, gradually revealing the side profile of the phone, showcasing its slim thickness and smooth edges. At 72-144°, the rotation continues, bringing the back of the phone into view, which is uniformly black and devoid of any markings. From 144-216°, the camera moves further around, offering a clear side view that highlights the phone's consistent proportions and design. In the next segment (216-288°), the camera completes the majority of the rotation, returning to the front view of the phone. Finally, from 288-360°, the camera finishes the 360-degree loop, mirroring the initial front view and confirming the phone's identity and consistency throughout the entire rotation." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.json +[02-20 14:57:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 14:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 14:57:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 14:57:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 15:10:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 15:10:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:10:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:10:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 15:10:56|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..32cc0256218ec9fa4376e8f184cedb20ca346d6b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the black smartphone on the wooden table, emphasizing its rectangular shape and rounded corners. From 0-72°, the camera begins its rotation to the right, gradually revealing the side profile of the phone, showcasing its slim thickness and smooth edges. At 72-144°, the rotation continues, bringing the back of the phone into view, which is uniformly black and devoid of any markings. From 144-216°, the camera moves further around, offering a clear side view that highlights the phone's consistent proportions and design. In the next segment (216-288°), the camera completes the majority of the rotation, returning to the front view of the phone. Finally, from 288-360°, the camera finishes the 360-degree loop, mirroring the initial front view and confirming the phone's identity and consistency throughout the entire rotation.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62d3178c9f81468ca0e7938e5fd09a9544d05242 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d68eb6c245565c7c4cdc3034d5c23c17de7e9be084ec23ba8df086305cd654d2 +size 598011 diff --git a/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/config.yaml b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f63449e6a97a1c044e9d7a6662e86b376112ab16 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/console.log b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/console.log new file mode 100644 index 0000000000000000000000000000000000000000..58467233b5c3871bac4879abf80f7195fab0b490 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/console.log @@ -0,0 +1,22 @@ +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:33:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:33:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/config.yaml +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.json +[02-20 14:33:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:47:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:47:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:47:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:48:00|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/debug.log b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ae0a310e30b9329256e45100aab8c625c5b42c0b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/debug.log @@ -0,0 +1,352 @@ +[02-20 14:32:47|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 14:32:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:32:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:32:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 14:32:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 14:32:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 14:32:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 14:32:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 14:32:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 14:32:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 14:32:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 14:32:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 14:32:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 14:32:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 14:32:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 14:32:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 14:32:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 14:32:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 14:32:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 14:32:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 14:32:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 14:32:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 14:32:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 14:32:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 14:32:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 14:32:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 14:32:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 14:32:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 14:32:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 14:32:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:33:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:33:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 14:33:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:33:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 14:33:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:33:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:33:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 14:33:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:33:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:33:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 14:33:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 14:33:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.38 s +[02-20 14:33:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:33:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:33:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 14:33:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/config.yaml +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 14:33:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a direct front view of the book, showcasing the red border at the top and the light blue central area adorned with an abstract design featuring red and blue elements. The title 'COMPUTER PROGRAMMING FOUNDATION' is clearly visible on the right side, rotated 90 degrees clockwise, while the author's name, 'Emmanuel ADETIBA,' is displayed vertically on the left. As the camera begins its rotation, moving from the front to the right side (0-72°), the red border transitions into the edge of the cover, and the abstract design becomes more prominent. By the time the camera reaches the side view (72-144°), the front cover is no longer visible, but the consistent color scheme and design elements are maintained. As the rotation continues to the back of the book (144-216°), the original front cover is no longer visible, but the abstract design and color scheme remain consistent. Finally, as the camera moves from the back to the front again (216-360°), the front cover reappears, completing the 360-degree rotation and reiterating the clarity and consistency of the book's appearance." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.json +[02-20 14:33:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 14:33:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 14:33:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 14:33:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 14:47:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 14:47:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:47:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:47:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:47:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 14:48:00|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3107c66534677986c8257bf1038115c6730db268 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a direct front view of the book, showcasing the red border at the top and the light blue central area adorned with an abstract design featuring red and blue elements. The title 'COMPUTER PROGRAMMING FOUNDATION' is clearly visible on the right side, rotated 90 degrees clockwise, while the author's name, 'Emmanuel ADETIBA,' is displayed vertically on the left. As the camera begins its rotation, moving from the front to the right side (0-72°), the red border transitions into the edge of the cover, and the abstract design becomes more prominent. By the time the camera reaches the side view (72-144°), the front cover is no longer visible, but the consistent color scheme and design elements are maintained. As the rotation continues to the back of the book (144-216°), the original front cover is no longer visible, but the abstract design and color scheme remain consistent. Finally, as the camera moves from the back to the front again (216-360°), the front cover reappears, completing the 360-degree rotation and reiterating the clarity and consistency of the book's appearance.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb4da112e5aeebe063730a35bbc7d922f20de72f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baebb7eea21e9abc17d540a580f8dfb22061bb15a5c005f0dcfb575249de0385 +size 709712 diff --git a/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/config.yaml b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..147ef3a6c0a4e9a1accd0d47d3e8c85d16c41897 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/console.log b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/console.log new file mode 100644 index 0000000000000000000000000000000000000000..cd17c77e965fee87ab931a7c48b3139ef7821196 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/console.log @@ -0,0 +1,22 @@ +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:24:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:24:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/config.yaml +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.json +[02-19 23:25:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:38:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:38:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:38:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:38:35|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/debug.log b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..fd65f43123d13112882ad7247fef9964a8711233 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/debug.log @@ -0,0 +1,352 @@ +[02-19 23:24:28|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 23:24:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:24:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 23:24:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 23:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 23:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 23:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 23:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 23:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 23:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 23:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 23:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 23:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 23:24:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 23:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 23:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 23:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 23:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 23:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 23:24:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 23:24:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 23:24:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 23:24:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 23:24:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 23:24:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 23:24:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 23:24:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 23:24:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 23:24:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 23:24:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 23:24:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 23:24:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 23:24:41|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 23:24:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 23:24:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:24:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:24:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 23:24:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:24:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:24:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:24:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 23:24:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:24:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 23:24:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:24:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:24:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 23:24:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:24:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 23:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 23:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.72 s +[02-19 23:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:25:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:25:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/config.yaml +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:25:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a direct front view of the red plastic cup, emphasizing its solid red color and cylindrical form. The camera then rotates 72 degrees to the right, offering a side perspective that showcases the cup's smooth, non-reflective surface. Continuing the rotation, the next 72 degrees reveal the back of the cup, maintaining the same red hue and shape. Another 72-degree turn brings the cup back to the front, now viewed from a slightly different angle. Finally, the last 72 degrees complete the 360-degree rotation, returning to the original front view. The entire sequence ensures the cup's appearance remains consistent and unaltered." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.json +[02-19 23:25:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 23:25:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 23:25:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 23:25:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:38:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:38:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:38:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:38:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:38:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:38:35|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..af913961ef509289b4f6a55d0c927ab80a2330ef --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the red plastic cup, emphasizing its solid red color and cylindrical form. The camera then rotates 72 degrees to the right, offering a side perspective that showcases the cup's smooth, non-reflective surface. Continuing the rotation, the next 72 degrees reveal the back of the cup, maintaining the same red hue and shape. Another 72-degree turn brings the cup back to the front, now viewed from a slightly different angle. Finally, the last 72 degrees complete the 360-degree rotation, returning to the original front view. The entire sequence ensures the cup's appearance remains consistent and unaltered.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a4c699f17781fac0266c47a544a39d4b906d17f6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28b8bab6c0e8fab46e9b247cb8acea21805b1c15f3c95af085bb887b19272361 +size 599411 diff --git a/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/config.yaml b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..954dfb22deb53f78e321b12e73355ef2854fb80e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/console.log b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/console.log new file mode 100644 index 0000000000000000000000000000000000000000..f870d357e5cf7d38b37e40dd19f3796ec7c4ece6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/console.log @@ -0,0 +1,22 @@ +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:50:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:50:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/config.yaml +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.json +[02-20 00:50:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:04:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:04:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:04:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:04:14|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/debug.log b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..aafa302b47d14b647cf1c0540afb124875e23184 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/debug.log @@ -0,0 +1,352 @@ +[02-20 00:50:08|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 00:50:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:50:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:50:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 00:50:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 00:50:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 00:50:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 00:50:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 00:50:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 00:50:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 00:50:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 00:50:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 00:50:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 00:50:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 00:50:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 00:50:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 00:50:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 00:50:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 00:50:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 00:50:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 00:50:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 00:50:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 00:50:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 00:50:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 00:50:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 00:50:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 00:50:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 00:50:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 00:50:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 00:50:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 00:50:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 00:50:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 00:50:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 00:50:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 00:50:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 00:50:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 00:50:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 00:50:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 00:50:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 00:50:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 00:50:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 00:50:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 00:50:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 00:50:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 00:50:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:50:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:50:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 00:50:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:50:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:50:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:50:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 00:50:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:50:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 00:50:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:50:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:50:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 00:50:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:50:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-20 00:50:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 00:50:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 00:50:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 00:50:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:50:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.67 s +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.68 s +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.68 s +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:50:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 00:50:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 00:50:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.73 s +[02-20 00:50:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:50:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:50:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 00:50:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/config.yaml +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 00:50:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a front view of the red apple, focusing on its glossy red skin, the small brown stem, and the green leaf. As the camera rotates 72 degrees to the right, the side of the apple comes into view, displaying the smooth texture and the subtle yellow gradient on the red skin. Rotating another 72 degrees, the camera shows the back of the apple, emphasizing the stem and leaf from a new perspective. Continuing the rotation, the camera moves to the left side of the apple, providing a clear view of the stem and the leaf's attachment point. Finally, as the camera completes the 360-degree rotation, it returns to the initial front view, ensuring the apple's identity is consistently preserved." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.json +[02-20 00:50:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 00:50:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 01:04:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 01:04:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:04:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:04:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:04:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 01:04:14|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..7348d8f3e5ef186fc99ec4b5ca82374839d7bad3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a front view of the red apple, focusing on its glossy red skin, the small brown stem, and the green leaf. As the camera rotates 72 degrees to the right, the side of the apple comes into view, displaying the smooth texture and the subtle yellow gradient on the red skin. Rotating another 72 degrees, the camera shows the back of the apple, emphasizing the stem and leaf from a new perspective. Continuing the rotation, the camera moves to the left side of the apple, providing a clear view of the stem and the leaf's attachment point. Finally, as the camera completes the 360-degree rotation, it returns to the initial front view, ensuring the apple's identity is consistently preserved.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0169aa47b1156303836b2481ee0b81ff7c939529 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cc096f5b87ab3e75e11d207cd8b2bb08f4716faf9989acd740fd0f58429ef11 +size 769178 diff --git a/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/config.yaml b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7ec9220346180c6858e8a5a5fcaee0d072d3dde --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/console.log b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b9e7f529674d71c77d8347f41eafc4c3282e4e33 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/console.log @@ -0,0 +1,22 @@ +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:46:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:46:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/config.yaml +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.json +[02-20 02:47:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:00:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:00:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:00:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:00:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/debug.log b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e77997e1f79c3c1b11249c613de498f6953b9233 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/debug.log @@ -0,0 +1,352 @@ +[02-20 02:46:27|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 02:46:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:46:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 02:46:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 02:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 02:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 02:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 02:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 02:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 02:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 02:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 02:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 02:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 02:46:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 02:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 02:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 02:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 02:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 02:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 02:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 02:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 02:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 02:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 02:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 02:46:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 02:46:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 02:46:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 02:46:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 02:46:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 02:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:46:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 02:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:46:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 02:46:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:46:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 02:46:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 02:46:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 02:46:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 02:46:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:46:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 02:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 02:46:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 02:46:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:46:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 02:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 02:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.33 s +[02-20 02:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:47:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:47:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/config.yaml +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 02:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a front view of the black Toyota Tacoma TRD Sport Edition, focusing on its chrome-accented grille with the Toyota logo, bright headlights, and silver multi-spoke alloy wheels. As the camera rotates 72 degrees to the left, the 'Tacoma' model name on the front fender becomes visible. Continuing another 72 degrees, the side profile is shown, revealing the 'TRD Sport Edition' badge on the rear side panel and the wooden planks in the truck bed. Rotating further 72 degrees to the right, the rear of the truck is captured, displaying the taillights and the 'TRD Sport Edition' badge. Finally, the camera completes the 360-degree rotation by returning to the front, ensuring all visual details align with the starting view." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.json +[02-20 02:47:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 02:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 02:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 02:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 03:00:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 03:00:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:00:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:00:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:00:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 03:00:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..37a22e3653d9f63dadb53f71367b37c67c1e5096 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a front view of the black Toyota Tacoma TRD Sport Edition, focusing on its chrome-accented grille with the Toyota logo, bright headlights, and silver multi-spoke alloy wheels. As the camera rotates 72 degrees to the left, the 'Tacoma' model name on the front fender becomes visible. Continuing another 72 degrees, the side profile is shown, revealing the 'TRD Sport Edition' badge on the rear side panel and the wooden planks in the truck bed. Rotating further 72 degrees to the right, the rear of the truck is captured, displaying the taillights and the 'TRD Sport Edition' badge. Finally, the camera completes the 360-degree rotation by returning to the front, ensuring all visual details align with the starting view.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b68a6a10e27bcb7b7a96ab33b6de59d3e665748d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d24bdcc968bafe317ccd62b3fd2e88d4867dfb58e46d053de97ccc087468fdd +size 1261529 diff --git a/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/config.yaml b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66a04c2f17d63ad8b17338f4fe15ae0406ddf205 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/console.log b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ae02bee45ae69a1c4cc0a9d38da9ad0faf8820b1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/console.log @@ -0,0 +1,22 @@ +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:08:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:08:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/config.yaml +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.json +[02-20 07:08:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:22:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:22:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/debug.log b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b0eec40206797bdf959bcbe4d0c69641dd88bbf7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/debug.log @@ -0,0 +1,352 @@ +[02-20 07:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 07:08:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:08:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 07:08:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 07:08:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 07:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 07:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 07:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 07:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 07:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 07:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 07:08:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 07:08:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 07:08:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 07:08:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 07:08:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 07:08:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 07:08:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 07:08:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 07:08:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 07:08:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 07:08:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 07:08:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 07:08:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 07:08:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 07:08:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 07:08:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 07:08:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 07:08:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:08:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:08:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:08:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 07:08:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:08:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 07:08:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:08:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:08:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 07:08:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:08:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 07:08:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 07:08:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 07:08:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 07:08:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:08:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:08:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 07:08:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 07:08:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.05 s +[02-20 07:08:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:08:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:08:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/config.yaml +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 07:08:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a front-facing view of the teal plastic chair, emphasizing the perforated backrest and the three elongated rectangular cutouts on the seat. The smooth, glossy texture and curved armrests are prominent. As the camera moves 72 degrees to the right, the side profile of the chair becomes visible, showcasing the slightly angled legs and the continuous teal color. Rotating another 72 degrees (144 degrees total), the chair's back is now in view, displaying the same perforated pattern and structural details. Continuing the rotation to 216 degrees, the left side of the chair is shown, maintaining the consistent appearance. Finally, rotating the last 144 degrees (288-360 degrees), the chair returns to the initial front view, completing the seamless 360-degree rotation." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.json +[02-20 07:08:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 07:08:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 07:08:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 07:08:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 07:22:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 07:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:22:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:22:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 07:22:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f4a4b72d25736a5c68515d563438ed2dab57dbcb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a front-facing view of the teal plastic chair, emphasizing the perforated backrest and the three elongated rectangular cutouts on the seat. The smooth, glossy texture and curved armrests are prominent. As the camera moves 72 degrees to the right, the side profile of the chair becomes visible, showcasing the slightly angled legs and the continuous teal color. Rotating another 72 degrees (144 degrees total), the chair's back is now in view, displaying the same perforated pattern and structural details. Continuing the rotation to 216 degrees, the left side of the chair is shown, maintaining the consistent appearance. Finally, rotating the last 144 degrees (288-360 degrees), the chair returns to the initial front view, completing the seamless 360-degree rotation.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..86a332764cfa1c7643e8d4556466c80de12d1978 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cf166bedd201976545c7e4fd301e191e3cbeec8454f45dca8ca0bad0e2cd07f +size 1816968 diff --git a/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/config.yaml b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff7fc7979a47fc9efe056347a77d6ebb468c1ec3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/console.log b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/console.log new file mode 100644 index 0000000000000000000000000000000000000000..6abc8a796d813b1fcd343f3acf70347e363fcb8a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/console.log @@ -0,0 +1,22 @@ +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:12:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:12:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/config.yaml +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.json +[02-19 23:13:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:26:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:26:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:26:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:26:37|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/debug.log b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b221b6b7ea787f6b9810c4c1f4fded6d2cddd919 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/debug.log @@ -0,0 +1,352 @@ +[02-19 23:12:23|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 23:12:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:12:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:12:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 23:12:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 23:12:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 23:12:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 23:12:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 23:12:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 23:12:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 23:12:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 23:12:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 23:12:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 23:12:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 23:12:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 23:12:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 23:12:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 23:12:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 23:12:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 23:12:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 23:12:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 23:12:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 23:12:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 23:12:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 23:12:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 23:12:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 23:12:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 23:12:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 23:12:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 23:12:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 23:12:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 23:12:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 23:12:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 23:12:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 23:12:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 23:12:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:12:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:12:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 23:12:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:12:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:12:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:12:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 23:12:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:12:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 23:12:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:12:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:12:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 23:12:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:12:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 23:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 23:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.91 s +[02-19 23:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:13:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:13:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/config.yaml +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video starts with a front view of the spherical puzzle toy, focusing on the green and yellow segments. As the camera rotates 72 degrees to the right, the blue and white segments become visible, showcasing their smooth, glossy texture. Rotating another 72 degrees (144 degrees total), the back of the toy is revealed, with the green and yellow segments appearing again in a symmetrical pattern. Another 72-degree rotation (216 degrees total) brings the side view back into focus, highlighting the blue and white segments once more. Finally, rotating the last 72 degrees (288-360 degrees), the camera returns to the initial front view, completing the 360-degree rotation with the object appearing identical to how it started.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.json +[02-19 23:13:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 23:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 23:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 23:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:26:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:26:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:26:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:26:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:26:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:26:37|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..87e60523038988e320ccfae82b2154b7c07807d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a front view of the spherical puzzle toy, focusing on the green and yellow segments. As the camera rotates 72 degrees to the right, the blue and white segments become visible, showcasing their smooth, glossy texture. Rotating another 72 degrees (144 degrees total), the back of the toy is revealed, with the green and yellow segments appearing again in a symmetrical pattern. Another 72-degree rotation (216 degrees total) brings the side view back into focus, highlighting the blue and white segments once more. Finally, rotating the last 72 degrees (288-360 degrees), the camera returns to the initial front view, completing the 360-degree rotation with the object appearing identical to how it started.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46ab9db63dd5e8d4844d7bebbd14ae102ede6826 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44262bd852cb41b56a5bfb5ffc51fc40fd03f1aa1427673b989890d8aa962e80 +size 711718 diff --git a/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/config.yaml b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..101e933343575213fa278738efdf7d9b0f3e06d8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/console.log b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/console.log new file mode 100644 index 0000000000000000000000000000000000000000..7030f342f40eb39b906c1a737d941b6fa2ca2cfd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/console.log @@ -0,0 +1,22 @@ +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/config.yaml +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.json +[02-20 10:41:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:54:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:54:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:54:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:54:57|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/debug.log b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..203178683ccca5016a86c7a9ac06703dc9389935 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/debug.log @@ -0,0 +1,352 @@ +[02-20 10:40:46|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 10:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:40:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 10:40:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 10:40:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 10:40:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 10:40:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 10:40:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 10:40:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 10:40:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 10:40:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 10:40:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 10:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 10:40:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 10:40:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 10:40:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 10:40:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 10:40:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 10:40:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 10:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 10:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 10:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 10:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 10:40:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 10:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 10:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 10:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 10:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:41:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 10:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 10:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:41:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:41:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:41:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 10:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 10:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.78 s +[02-20 10:41:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:41:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:41:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 10:41:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/config.yaml +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 10:41:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='Starting at the 0-degree angle, the video shows the front view of the bowl, with the colorful central dots clearly visible against the white background. As the camera moves through the first 72 degrees (0-72°), the left side of the bowl becomes more prominent, emphasizing the scalloped edge and the floral motifs. By the 144-degree mark (72-144°), the camera has rotated to the side, offering a profile view that highlights the depth and curvature of the bowl, as well as the detailed texture of the floral border. At 216 degrees (144-216°), the back of the bowl is revealed, showing the continuation of the floral pattern and the subtle sheen of the glossy surface. Moving through the next 72 degrees (216-288°), the camera returns to the front, now slightly angled, providing a dynamic perspective of the central dots and the edge details. Finally, from 288 to 360 degrees (288-360°), the camera completes the rotation, returning to the original front view, ensuring that all visual elements match perfectly.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.json +[02-20 10:41:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 10:41:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 10:41:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 10:41:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 10:54:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 10:54:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:54:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:54:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:54:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 10:54:57|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f3e0b6a16a5ac8f9024899478f0c8ea8953d28c3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Starting at the 0-degree angle, the video shows the front view of the bowl, with the colorful central dots clearly visible against the white background. As the camera moves through the first 72 degrees (0-72°), the left side of the bowl becomes more prominent, emphasizing the scalloped edge and the floral motifs. By the 144-degree mark (72-144°), the camera has rotated to the side, offering a profile view that highlights the depth and curvature of the bowl, as well as the detailed texture of the floral border. At 216 degrees (144-216°), the back of the bowl is revealed, showing the continuation of the floral pattern and the subtle sheen of the glossy surface. Moving through the next 72 degrees (216-288°), the camera returns to the front, now slightly angled, providing a dynamic perspective of the central dots and the edge details. Finally, from 288 to 360 degrees (288-360°), the camera completes the rotation, returning to the original front view, ensuring that all visual elements match perfectly.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c4da0622a5e95fe1a3167e0c5b1e20d9eed283c4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5916a79ecace7a626864dd567508630913898bb42305aab0fe9af3e222f63995 +size 791723 diff --git a/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/config.yaml b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..281187cc4766d2fde3835e3543957572f69b7860 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/console.log b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c005ba9ca8b0f2c63775c1e29ff21c02ab5e0b29 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/console.log @@ -0,0 +1,22 @@ +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:37:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:37:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/config.yaml +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.json +[02-19 21:38:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:52:40|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/debug.log b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ad3ab55f54815f1118e62ccb347e6bd2118e6d69 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/debug.log @@ -0,0 +1,352 @@ +[02-19 21:37:31|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 21:37:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:37:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 21:37:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 21:37:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 21:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 21:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 21:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 21:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 21:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 21:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 21:37:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 21:37:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 21:37:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 21:37:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 21:37:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 21:37:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 21:37:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 21:37:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 21:37:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 21:37:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 21:37:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 21:37:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 21:37:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 21:37:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 21:37:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 21:37:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 21:37:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 21:37:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 21:37:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 21:37:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 21:37:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 21:37:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 21:37:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 21:37:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:37:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:37:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 21:37:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:37:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:37:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:37:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 21:37:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 21:37:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:37:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 21:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-19 21:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 21:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 21:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 21:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.65 s +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.65 s +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:37:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:38:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 21:38:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 21:38:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.61 s +[02-19 21:38:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:38:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:38:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 21:38:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/config.yaml +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 21:38:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a direct front view of the Wilson Ultra basketball, highlighting its brown leather texture, black panel lines, and the prominent 'Wilson' and 'ULTRA' logos. As the camera moves 72 degrees to the right, the side of the ball comes into view, revealing the small circular valve and the continued texture of the leather. Another 72-degree rotation brings us to the back of the ball, where the mirrored logos are visible against the same worn surface. Continuing another 72 degrees, the left side of the ball is shown, emphasizing the texture and the valve once more. Finally, the last 72-degree rotation returns the ball to the front view, completing the 360-degree loop with the object appearing identical to its starting state." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.json +[02-19 21:38:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 21:38:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 21:38:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 21:38:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:52:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 21:52:40|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..1835d1d78dfaad622c897d56884836ccdc3277e7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the Wilson Ultra basketball, highlighting its brown leather texture, black panel lines, and the prominent 'Wilson' and 'ULTRA' logos. As the camera moves 72 degrees to the right, the side of the ball comes into view, revealing the small circular valve and the continued texture of the leather. Another 72-degree rotation brings us to the back of the ball, where the mirrored logos are visible against the same worn surface. Continuing another 72 degrees, the left side of the ball is shown, emphasizing the texture and the valve once more. Finally, the last 72-degree rotation returns the ball to the front view, completing the 360-degree loop with the object appearing identical to its starting state.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2088d7efa973a812743ad620d8fa22854ee2aae8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce185ef5aba3f9692c35fc49b510836c84b2ccd35a5f806ff8f746787558c8f0 +size 1439797 diff --git a/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/config.yaml b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5c280532ee9ad97db50d293e5ebfab31fdcc4a7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/console.log b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/console.log new file mode 100644 index 0000000000000000000000000000000000000000..eb16d9cd8662d4c32999cb43b5a456c76dd2d3e5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/console.log @@ -0,0 +1,22 @@ +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/config.yaml +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.json +[02-20 01:19:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:33:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:33:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:33:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:33:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/debug.log b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bcae5fd102919109d0778258362df5f7ff93b100 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/debug.log @@ -0,0 +1,352 @@ +[02-20 01:19:03|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 01:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:19:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 01:19:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 01:19:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 01:19:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 01:19:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 01:19:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 01:19:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 01:19:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 01:19:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 01:19:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 01:19:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 01:19:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 01:19:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 01:19:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 01:19:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 01:19:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 01:19:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 01:19:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 01:19:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 01:19:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 01:19:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 01:19:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 01:19:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 01:19:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 01:19:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 01:19:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 01:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 01:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:19:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 01:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:19:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 01:19:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:19:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 0.92 s +[02-20 01:19:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 01:19:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 01:19:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 01:19:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 01:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 01:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.61 s +[02-20 01:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:19:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:19:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/config.yaml +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 01:19:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a front view of two carrot-like objects, highlighting their bright orange color and smooth, matte texture. The objects are elongated and slightly irregular in shape, with one passing through the other. As the camera rotates 72 degrees (0-72°), the side profile becomes visible, showing the interwoven structure and natural variations in hue. Continuing to rotate another 72 degrees (72-144°), the objects are viewed from a different angle, further emphasizing their consistent shape and texture. Rotating another 72 degrees (144-216°), the back view reveals the uniformity of the objects' appearance. Another 72 degrees (216-288°) brings the objects closer to the initial front view, with the interwoven structure clearly visible. Finally, rotating the last 72 degrees (288-360°), the camera returns to the starting point, completing the 360-degree rotation and confirming the identity persistence of the objects." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.json +[02-20 01:19:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 01:19:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 01:19:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 01:19:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 01:33:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 01:33:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:33:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:33:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 01:33:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..b04b1ca9dc4ef1bc172e5f1c1f3ecddced7d92a8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a front view of two carrot-like objects, highlighting their bright orange color and smooth, matte texture. The objects are elongated and slightly irregular in shape, with one passing through the other. As the camera rotates 72 degrees (0-72°), the side profile becomes visible, showing the interwoven structure and natural variations in hue. Continuing to rotate another 72 degrees (72-144°), the objects are viewed from a different angle, further emphasizing their consistent shape and texture. Rotating another 72 degrees (144-216°), the back view reveals the uniformity of the objects' appearance. Another 72 degrees (216-288°) brings the objects closer to the initial front view, with the interwoven structure clearly visible. Finally, rotating the last 72 degrees (288-360°), the camera returns to the starting point, completing the 360-degree rotation and confirming the identity persistence of the objects.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ffb0e224aa08fd11d8395b21f04aa24599002a07 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96996e933aa3c8fd9156d3be83a55a69d9f97586bdb9842247a9ebe3f978e4ce +size 780599 diff --git a/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/config.yaml b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e92b32f3fef8370049a4a6a9bf52bfb7aa8582a0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/console.log b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b46a1274015ceac084f36fdf0b795817c73093ea --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/console.log @@ -0,0 +1,22 @@ +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/config.yaml +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.json +[02-20 12:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:15:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:15:38|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/debug.log b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a8f9e30d40c3165892b3645242fcc8f439d3ab8d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/debug.log @@ -0,0 +1,352 @@ +[02-20 12:01:30|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 12:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:01:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 12:01:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 12:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 12:01:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 12:01:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 12:01:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 12:01:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 12:01:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 12:01:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 12:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 12:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 12:01:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 12:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 12:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 12:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 12:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 12:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 12:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 12:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 12:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 12:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 12:01:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 12:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 12:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 12:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 12:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 12:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 12:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 12:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 12:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 12:01:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 12:01:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 12:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 12:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 12:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:01:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 12:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:01:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 12:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:01:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 12:01:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:01:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 12:01:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 12:01:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 12:01:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 12:01:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:01:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:02:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 12:02:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 12:02:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.16 s +[02-20 12:02:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:02:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:02:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/config.yaml +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 12:02:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video starts with a direct front view of the white baseball, focusing on the red stitched seams that form a symmetrical pattern on the smooth white leather surface. The camera then rotates 72 degrees to the right, offering a side perspective that reveals the textured gray and beige houndstooth fabric and a subtle shadow beneath the ball, emphasizing its spherical shape. Continuing the rotation, the next 72 degrees (144-216°) brings the back of the baseball into view, where the stitching pattern remains clearly visible. Another 72-degree turn (216-288°) returns the camera to the front-left angle, showing the ball from a slightly different perspective while maintaining the focus on its consistent size and shape. Finally, the last 72 degrees (288-360°) complete the rotation, bringing the view back to the original front position, ensuring the baseball appears identical to its starting point.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.json +[02-20 12:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 12:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 12:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 12:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 12:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 12:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:15:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:15:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 12:15:38|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c80cda23750b9972cd889c75b38f0cd4bc781386 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a direct front view of the white baseball, focusing on the red stitched seams that form a symmetrical pattern on the smooth white leather surface. The camera then rotates 72 degrees to the right, offering a side perspective that reveals the textured gray and beige houndstooth fabric and a subtle shadow beneath the ball, emphasizing its spherical shape. Continuing the rotation, the next 72 degrees (144-216°) brings the back of the baseball into view, where the stitching pattern remains clearly visible. Another 72-degree turn (216-288°) returns the camera to the front-left angle, showing the ball from a slightly different perspective while maintaining the focus on its consistent size and shape. Finally, the last 72 degrees (288-360°) complete the rotation, bringing the view back to the original front position, ensuring the baseball appears identical to its starting point.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7c55359759fd59f46b07d6ef0250ea5a4e9451b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c1cc0b4538e79c0567f011beba6acdd6dbc255ce93d1726dca145942221e085 +size 1479486 diff --git a/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/config.yaml b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f5df01c061212d982baacf612be52868cf0bcf9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/console.log b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/console.log new file mode 100644 index 0000000000000000000000000000000000000000..5f44dce04e14149c4f6c7c95783e407aad25ffd6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/console.log @@ -0,0 +1,22 @@ +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/config.yaml +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.json +[02-20 04:30:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:43:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:43:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/debug.log b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..06e7a4abe38a0fef912bef4db0c8fd15e9d412a6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/debug.log @@ -0,0 +1,352 @@ +[02-20 04:29:37|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 04:29:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:29:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:29:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 04:29:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 04:29:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 04:29:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 04:29:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 04:29:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 04:29:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 04:29:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 04:29:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 04:29:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 04:29:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 04:29:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 04:29:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 04:29:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 04:29:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 04:29:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 04:29:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 04:29:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 04:29:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 04:29:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 04:29:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 04:29:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 04:29:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 04:29:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 04:29:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 04:29:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 04:29:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 04:29:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 04:29:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 04:29:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 04:29:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 04:29:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 04:29:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:29:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:29:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 04:29:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:29:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 04:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:29:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 04:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:29:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:29:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:29:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 04:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 04:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.03 s +[02-20 04:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:30:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:30:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/config.yaml +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 04:30:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a front view of the red and black Puma backpack on a black plastic stool. The backpack's red body, black accents, white leaping cat logo, and white stitching are immediately noticeable. From 0-72°, the camera begins its rotation to the right, gradually revealing the side profile of the backpack. The black straps and the compact, rounded shape become more prominent as the front view shifts slightly to the left. By 72-144°, the side of the backpack is fully in view, with the black straps and the edge of the red body clearly visible. The front is now seen from a slight angle, showing the logo and stitching. From 144-216°, the camera continues rotating, moving past the side and starting to show the back of the backpack. The back features zippers and possibly a mesh pocket, while the red and black color scheme remains consistent. By 216-288°, the back of the backpack is fully visible, with the zippers and mesh pocket clearly shown. The front view is now seen from the opposite side, still displaying the logo and stitching. Finally, from 288-360°, the camera completes the rotation, bringing the view back to the front of the backpack. The entire 360-degree journey ensures that the backpack's identity—its colors, logo, and shape—is preserved from every angle, concluding with the same front view as the start." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.json +[02-20 04:30:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 04:30:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 04:30:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 04:30:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 04:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 04:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:43:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:43:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 04:43:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..551335caec9d02aa391578206a9116f37d218d22 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a front view of the red and black Puma backpack on a black plastic stool. The backpack's red body, black accents, white leaping cat logo, and white stitching are immediately noticeable. From 0-72°, the camera begins its rotation to the right, gradually revealing the side profile of the backpack. The black straps and the compact, rounded shape become more prominent as the front view shifts slightly to the left. By 72-144°, the side of the backpack is fully in view, with the black straps and the edge of the red body clearly visible. The front is now seen from a slight angle, showing the logo and stitching. From 144-216°, the camera continues rotating, moving past the side and starting to show the back of the backpack. The back features zippers and possibly a mesh pocket, while the red and black color scheme remains consistent. By 216-288°, the back of the backpack is fully visible, with the zippers and mesh pocket clearly shown. The front view is now seen from the opposite side, still displaying the logo and stitching. Finally, from 288-360°, the camera completes the rotation, bringing the view back to the front of the backpack. The entire 360-degree journey ensures that the backpack's identity—its colors, logo, and shape—is preserved from every angle, concluding with the same front view as the start.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..de8c6b3e1e537a488d0aeadec3fbda47c8760971 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d01221098ba7e20506ae5f4acd745bcc56164231759e45ddb4cd5c875cb3ba7 +size 755582 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/config.yaml b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e633ad614e131c99d06a21cad57c4204b71f024 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/console.log b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/console.log new file mode 100644 index 0000000000000000000000000000000000000000..930c66f2bee1cfc5ac7a379400f812fdb8302d47 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/console.log @@ -0,0 +1,22 @@ +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/config.yaml +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.json +[02-20 15:50:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:04:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:04:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:04:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:04:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/debug.log b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..73a0907164b785696b0c9bb8e0e0033a6c7b9ec5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/debug.log @@ -0,0 +1,353 @@ +[02-20 15:49:36|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 15:49:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 15:49:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 15:49:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 15:49:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 15:49:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 15:49:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 15:49:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 15:49:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 15:49:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 15:49:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 15:49:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 15:49:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 15:49:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 15:49:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 15:49:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 15:49:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 15:49:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 15:49:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 15:49:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 15:49:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 15:49:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 15:49:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 15:49:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 15:49:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 15:49:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 15:49:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 15:49:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 15:49:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 15:49:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 15:49:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 15:49:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 15:49:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 15:49:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 15:49:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 15:49:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 15:49:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 15:49:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 15:49:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 15:49:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 15:49:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 15:49:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 15:49:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:49:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 15:49:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 15:49:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 15:49:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 15:49:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 15:49:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 15:49:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 15:49:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:49:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.15 s +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.69 s +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.70 s +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.70 s +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 15:49:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 15:49:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:50:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 15:50:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 15:50:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.00 s +[02-20 15:50:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:50:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:50:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 15:50:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/config.yaml +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 15:50:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, as the camera reaches 288 degrees, the front view is once again visible, completing the 360-degree rotation and ensuring the backpack appears exactly as it did at the start.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.json +[02-20 15:50:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.mp4 +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 15:50:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 15:50:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 15:50:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 16:04:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 16:04:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:04:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:04:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 16:04:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e51528df1346a4cd0bbfacd5de193a42e68bafa4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a clear front view of the gray heathered fabric backpack, focusing on the horizontal black zipper and the light brown top handle.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4c52afbe5ca452006ac836929668916b50085d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eec5d7b67c9e80943ea279ccab2b72e661bdbab26c3a67a50f37c4b70cc9747f +size 2269466 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_1.json b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..1ed9c6c7e530d1d78dc8874e05a359c8b2db3d35 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera rotates 72 degrees clockwise, the right side of the backpack becomes visible, showcasing the black shoulder straps and the smooth texture of the fabric.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_1.mp4 b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b083328e6c18814cc9e39a518352e7702f4eb36f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4587e13abcb44b3c8bf7ab6c3425b7440cce036e462689220c7ce84821f78e6d +size 1630481 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_2.json b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..371ce4f0e91535d535b843b3f2f2cba58eacdf9c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Continuing the rotation, at 144 degrees, the profile of the backpack is captured, emphasizing its depth and the consistent material.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_2.mp4 b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a4533fd184eeac0dc726b7ef0454bd37f68c7030 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20365795658992dbc07b16b4c9b3bb5b78ee9720bdb589193ba99b19ed684c7f +size 1202478 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.json b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..7f6ca3e7a29ef82f9f369d44a730d0b8d450ed13 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"By 216 degrees, the back of the backpack is shown, revealing the strap attachment points and the overall structure.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.mp4 b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8fcdfcc54f1569bca71a652e98c9135d4d2cc8b9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c49a2eb38c1af13ade23748e03485238fb4d67106ee36399eed6faa86e46f2a +size 1006502 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.json b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..14a516cde06fc088fff5c9f327e764473f6fb89e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, as the camera reaches 288 degrees, the front view is once again visible, completing the 360-degree rotation and ensuring the backpack appears exactly as it did at the start.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.mp4 b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58e0ab482b20b9bc51ae470ebcacc6cbb116d176 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:824df58d333837a263082d4bf35d372109bfe1ace6f934e74d0dd2644c88d88e +size 881473 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/config.yaml b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61bc3da182018880ec70da54909167d930436a96 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/console.log b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/console.log new file mode 100644 index 0000000000000000000000000000000000000000..9ed145b3676eab9f83d7654e63b1a9a711bd61dc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/console.log @@ -0,0 +1,22 @@ +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:27:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:27:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:27:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:27:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/config.yaml +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.json +[02-19 23:27:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:41:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:41:02|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/debug.log b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e55b728fd94ee70721ae5a9888882fc0563c6b80 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/debug.log @@ -0,0 +1,352 @@ +[02-19 23:26:48|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 23:26:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:26:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 23:26:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 23:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 23:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 23:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 23:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 23:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 23:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 23:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 23:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 23:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 23:26:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 23:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 23:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 23:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 23:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 23:26:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 23:26:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 23:26:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 23:26:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 23:26:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 23:26:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 23:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 23:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 23:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 23:27:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 23:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 23:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:27:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:27:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 23:27:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:27:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 23:27:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:27:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:27:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 23:27:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:27:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-19 23:27:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 23:27:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 23:27:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 23:27:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:27:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 23:27:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 23:27:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 23:27:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 23:27:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 23:27:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:27:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 23:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 23:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.63 s +[02-19 23:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:27:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:27:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/config.yaml +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 23:27:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a direct front view of the two 'Sweet Valley High' books on a wooden stool. The top book's purple cover with red and white text and the image of two blonde girls is prominent, while the bottom book's blue and brown cover with the title 'Nightmare in Death Valley' is partially visible. As the camera rotates 72 degrees to the right, the side profile of the books is revealed, showing the thickness of the stack and the spine of the bottom book with '#116.' Continuing the rotation, at 144 degrees, the back of the books is shown, displaying the continuation of the desert scene from the bottom book's cover. At 216 degrees, the camera focuses on the left side of the books, where the reflective glossy surface catches the light, creating bright highlights. Finally, as the camera rotates another 72 degrees to complete the 360-degree journey, the front view of the books is restored, maintaining consistency in appearance and detail." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.json +[02-19 23:27:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 23:27:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 23:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 23:27:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:41:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:41:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:41:02|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.json b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0a401330a325d56c952caf27c784c9a4d725f7e4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a direct front view of the two 'Sweet Valley High' books on a wooden stool. The top book's purple cover with red and white text and the image of two blonde girls is prominent, while the bottom book's blue and brown cover with the title 'Nightmare in Death Valley' is partially visible. As the camera rotates 72 degrees to the right, the side profile of the books is revealed, showing the thickness of the stack and the spine of the bottom book with '#116.' Continuing the rotation, at 144 degrees, the back of the books is shown, displaying the continuation of the desert scene from the bottom book's cover. At 216 degrees, the camera focuses on the left side of the books, where the reflective glossy surface catches the light, creating bright highlights. Finally, as the camera rotates another 72 degrees to complete the 360-degree journey, the front view of the books is restored, maintaining consistency in appearance and detail.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.mp4 b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d2fba7b97f2417485f6679dbeae0800dd353088 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc97137ba246b769e727f492ab5bb291ec33a12c53c87666e9057a6bad65953b +size 999106 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..080be2501b4198cf2668eddc5ef8376f7d4c126b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/console.log b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..74e9f2db68d9cd22e53de263420a09a0e835b690 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/console.log @@ -0,0 +1,22 @@ +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:15:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:15:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:15:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:15:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/config.yaml +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.json +[02-20 02:16:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:29:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:29:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cf3ee96596d95a92872f0610a9fe4112c8e335ab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/debug.log @@ -0,0 +1,353 @@ +[02-20 02:15:21|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 02:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:15:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 02:15:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 02:15:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 02:15:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 02:15:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 02:15:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 02:15:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 02:15:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 02:15:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 02:15:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 02:15:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 02:15:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 02:15:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 02:15:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 02:15:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 02:15:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 02:15:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 02:15:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 02:15:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 02:15:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 02:15:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 02:15:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 02:15:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 02:15:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 02:15:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 02:15:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 02:15:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 02:15:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 02:15:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 02:15:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 02:15:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 02:15:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 02:15:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:15:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 02:15:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:15:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:15:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 02:15:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 02:15:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:15:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 02:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:15:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 02:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:15:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 02:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 02:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 02:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 02:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 02:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:15:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:15:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.86 s +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:16:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 02:16:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/config.yaml +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 02:16:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera returns to the initial starting angle, completing the circular pan. The ironing board with its floral-patterned cover is centered, flanked by the beige sectional sofa on the left and the armchair with the orange pillow on the right. The TV displaying the cityscape, black TV stand, kitchen counter in the background, floor lamp near the window, and wall-mounted framed photos are all visually reaffirmed in their spatial positions, ensuring full continuity and completeness of the scene.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.json +[02-20 02:16:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.mp4 +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 02:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 02:16:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 02:16:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 02:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 02:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:29:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:29:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 02:29:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..9e5ba199e7144adbc9080d284f103fb9b94b0dd7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle shot focused on a white metal ironing board centered in the frame, covered with a floral-patterned fabric. To the left, a beige sectional sofa with a patterned blanket and throw pillows is partially visible. Behind it, a large flat-screen TV on a black TV stand displays a cityscape. Wall-mounted framed photos appear above the sofa. The camera begins a slow clockwise pan.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..32a23f1c39c837d83aa98dc85e6fddde55037c44 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ade41add847dc293c8e12204121638a0b60d21bd9080f6921eaf3bb7f74d804 +size 1467727 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9e6a9034eae2137e461becd66d32e0a9c70673ac --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the pan, the camera reveals more of the beige sectional sofa and the TV displaying the cityscape. The black TV stand remains visible beneath. The ironing board stays centered, its legs and floral cover detailed. The camera angle tilts slightly upward, capturing the top edge of the ironing board and the framed photos more clearly.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7196370befc30a51871d7f0614fa59e1bda50aa7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c3966bb871ba95289ea4edc22ec62d3f28f83b579633fd3d914b70b094a8468 +size 1271020 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..87f11bde837d1fe345b37c3a76e90ae137c4227e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera moves further clockwise, bringing a beige armchair with an orange pillow into view on the right. A tall floor lamp near a window with sheer curtains appears beside the armchair. The ironing board remains central, its floral pattern prominent. The kitchen counter in the background begins to come into view through the open-plan layout.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4e4976e9e0d3f698cbc7aee5e4c3aea91cd18161 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78e488a4d5bd261d5d04420c0909e720c967e7969a73162368353cad71fbb9a4 +size 1043197 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..2e9d077c8da122d198bb9c195baf2235a7bd0534 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The pan continues, now fully showcasing the armchair with the orange pillow and the floor lamp near the window. The kitchen counter in the background is clearly visible, showing bar stools and modern appliances. The ironing board’s structure and floral cover remain the visual anchor. The camera tilts slightly down to emphasize the base of the ironing board and the carpeted floor.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..378696723280bd63fc238a0ec9c7d132694b0529 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:feacb27f244221a1ce3cda02ebbcb8cb5f737bcea6c41d22dce387f86f4d6f3e +size 1614186 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..836c90039c7f1f9ac04709533f08ef972ec6addc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera pans further, now revealing the backside of the ironing board. The beige sectional sofa reappears on the left, its patterned blanket and pillows visible again. The TV displaying the cityscape and the black TV stand are seen from a new angle. The wall-mounted framed photos remain in view above the sofa.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f6546ace92dc8fd58bb0d9aed1b285db5629f948 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64ba456499cdf40feefd1f44c430017775c318aa8ca0470f2e76c27ea8c332bd +size 1694727 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..0481528592084d8c6097cb0be426d76c857e9b98 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: Continuing the circular motion, the camera captures the ironing board from a rear-three-quarter view. The kitchen counter is still visible in the background, and the armchair with the orange pillow is now seen from behind. The floor lamp near the window remains in frame. The camera maintains steady focus on the ironing board’s floral cover and metallic legs.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61020b733e127b973add3ba15d134a87c3baf705 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:204845e5454780534fc1afbf8ecb77359c739ee5f5ed58cc7b1a2f0dbb16fae2 +size 2174880 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..308dd0a282c28a074300b664d9514100e9679cec --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera nears completion of its 360-degree orbit, bringing the front of the ironing board back into view. The beige sectional sofa dominates the left side of the frame, with its patterned blanket and pillows clearly displayed. The TV on the black TV stand continues to show the cityscape. The wall-mounted framed photos are once again aligned above the sofa.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1eb64df0369c2ac16eb469c435e9c920c37672f2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31437b58135305f8301f69c0c449e79832a45f223d01274f5dec3418c906b881 +size 1966024 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..a7c57efdfe5d3b3f55eb24fd727f2ec054eb4b24 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera returns to the initial starting angle, completing the circular pan. The ironing board with its floral-patterned cover is centered, flanked by the beige sectional sofa on the left and the armchair with the orange pillow on the right. The TV displaying the cityscape, black TV stand, kitchen counter in the background, floor lamp near the window, and wall-mounted framed photos are all visually reaffirmed in their spatial positions, ensuring full continuity and completeness of the scene.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e34a1d332dae749808d809b1d9f2ff90527cad00 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57e97a3d0748b7a8f766fb80a326bfc1db6bcbbb5a4b35f3c1c0e02a5a2a70f8 +size 1618098 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de836570149cf8defcecee18c64576f3dd3d6bd8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/console.log b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/console.log new file mode 100644 index 0000000000000000000000000000000000000000..215c069655cd5eb750f5a1337baabb29abf4ca97 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/console.log @@ -0,0 +1,22 @@ +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:51:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:51:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/config.yaml +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.json +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:05:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:05:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:05:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:05:56|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..755a3f11c425fd8a9242df7e90182c84fc215c58 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/debug.log @@ -0,0 +1,353 @@ +[02-20 13:51:41|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 13:51:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:51:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:51:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 13:51:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 13:51:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 13:51:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 13:51:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 13:51:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 13:51:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 13:51:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 13:51:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 13:51:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 13:51:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 13:51:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 13:51:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 13:51:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 13:51:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 13:51:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 13:51:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 13:51:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 13:51:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 13:51:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 13:51:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 13:51:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 13:51:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 13:51:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 13:51:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 13:51:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 13:51:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 13:51:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 13:51:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 13:51:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 13:51:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 13:51:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 13:51:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:51:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:51:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 13:51:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:51:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:51:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:51:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 13:51:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:51:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 13:51:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:51:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:51:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 13:51:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:51:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 13:51:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 13:51:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 13:51:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 13:51:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:51:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:51:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 13:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 13:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.09 s +[02-20 13:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:52:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:52:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/config.yaml +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 13:52:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its orbit, returning to a composition similar to the opening shot. The black office chair, still spinning slowly, centers the frame with the patterned blanket draped over it. All mandatory landmarks — red office chair, TV displaying cityscape, carpeted floor, blanket draped over chair, yellow sofas, chair base with wheels, wall art, exercise ball, TV on stand — are clearly visible in their original spatial relationships, ensuring perfect continuity and closure.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.json +[02-20 13:52:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:52:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.mp4 +[02-20 13:52:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 13:52:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 13:52:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 13:52:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 14:05:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 14:05:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:05:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:05:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:05:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 14:05:56|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..1262c06e038e3cffe7da538be679eaca89a7e73f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle shot centered on a black office chair with a patterned blanket draped over its back. The chair sits on a beige carpeted floor. In the background, a red office chair and a large TV displaying a cityscape are visible to the left. A yellow sofa with a matching patterned pillow occupies the right side of the frame.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ee3c9bee999541fa28b5bbc118e5653d36a28278 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:944f0e9d009d5340f3263d9d94332ffbff9ac5cd50ad0e7b5ce53beff5eebccb +size 760602 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5dbf89603276674e97c1c5d3f62cda00ee077cbb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera begins a slow clockwise rotation around the chair. As it moves, the second yellow sofa comes into view opposite the first. Wall art — framed pictures arranged neatly — appears on the back wall. The blanket drapes loosely over the chair’s backrest, its geometric pattern visible under soft indoor lighting.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..629c5ff415d0f272ebb62a927af869d6798be625 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:295f5f21021690816312c1dd4f123e35468a2273d0319a2f425c5fcb26d97f48 +size 1743250 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..8a2b25fabb5f18a26c9a01c43e9364b091ff953d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: Continuing the rotation, the camera reveals more of the room’s layout: the TV on its stand remains in view, displaying the cityscape. The red office chair is now fully visible beside the TV. The beige carpet extends uniformly beneath all furniture. The chair’s base with five wheeled casters is briefly glimpsed as the camera dips slightly.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e4cb9d037d4c4b991d68df508aa39e828b88189e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7009decdc069e8f8557679e66e4e85d8633d1feb8fb8f0ef840f145eca310b39 +size 1043676 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..75a1d45eeee9dc16b4b11042a6868e1c78ceaedd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera tilts down sharply to focus on the chair’s base — the black hydraulic stem and star-shaped base with wheels rolling slightly on the carpet. The underside of the seat and adjustment lever are momentarily visible. The yellow sofas and red chair blur softly in the background due to shallow depth of field.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2cae552bc50da721c89576957c33e5e48c618c22 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2396640533a8b89f99a031e688b2ce6d712cbd9228234fead44ba91ff9e6334 +size 874006 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..92ea8365a96b1a9c6c4323099a003baa874e741b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: Rising again, the camera captures the blanket billowing gently as the chair spins. Its cream-colored fabric with muted circular patterns catches the light. The TV screen remains active, showing the cityscape, now partially hidden behind the rotating chair. Wall art and both yellow sofas frame the scene symmetrically.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8fbe149113cf51b49bfbb8487797ab9b72e84f17 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40be3506ac1b5c9c02923a29f6ccf8ec921bb7a216d148738d1913aae6b7629f +size 959454 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9003a1a7fb43344c9cb3afc096bc7acf0bd9e811 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera circles further, bringing the exercise ball into view near the right yellow sofa. The red office chair remains stationary beside the TV. The blanket’s drape shifts dynamically with the chair’s motion. The camera angle lowers again, refocusing on the chair’s wheelbase and the texture of the carpet beneath.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..563c88a86d28602dbece77c001d68c1e13f67ef8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52c49683dbadd0a6e0a795c1a83e02b43249ca2284c9cd5834f5c193e7086de4 +size 972427 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..c9aad95090429e93110693751e397cea9d5a2f44 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera ascends for a wider perspective, capturing the entire living room: the two yellow sofas facing each other, the red office chair, the TV on its stand with the cityscape, and the wall art. The blanket continues to flow with the chair’s rotation. Ambient lighting highlights the room’s warm, lived-in atmosphere.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a03008cb465b770088da06482f815ce32efac367 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6680aca8899d2aa2e5a5e983e58337883e467d8376bf7ee81851a0994c9c59ff +size 1325278 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..79f27d0f401da4cf40522bafaf028554f35fd146 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its orbit, returning to a composition similar to the opening shot. The black office chair, still spinning slowly, centers the frame with the patterned blanket draped over it. All mandatory landmarks — red office chair, TV displaying cityscape, carpeted floor, blanket draped over chair, yellow sofas, chair base with wheels, wall art, exercise ball, TV on stand — are clearly visible in their original spatial relationships, ensuring perfect continuity and closure.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15c9f9582fb3b907d6f3e8ae42bdc23ae5cf5796 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17071fa543533d7de7404190779eb05d7fe10745c5354f59f00751cd72c66c81 +size 1114424 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f71c4bef38875f387759f25349cd30a54f6c45d4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/console.log b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/console.log new file mode 100644 index 0000000000000000000000000000000000000000..5a1db94ec1c664f0751271aa0400282ce0551946 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/console.log @@ -0,0 +1,22 @@ +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 23:27:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 23:27:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/config.yaml +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.json +[02-20 23:28:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 23:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 23:41:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 23:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 23:41:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..22d110b217ee5ba63282b42e425f7426a021e34d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/debug.log @@ -0,0 +1,353 @@ +[02-20 23:27:42|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 23:27:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 23:27:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 23:27:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 23:27:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 23:27:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 23:27:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 23:27:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 23:27:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 23:27:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 23:27:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 23:27:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 23:27:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 23:27:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 23:27:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 23:27:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 23:27:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 23:27:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 23:27:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 23:27:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 23:27:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 23:27:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 23:27:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 23:27:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 23:27:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 23:27:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 23:27:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 23:27:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 23:27:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 23:27:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 23:27:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 23:27:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 23:27:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 23:27:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 23:27:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 23:27:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 23:27:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 23:27:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 23:27:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 23:27:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 23:27:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 23:27:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 23:27:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 23:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 23:27:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 23:27:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 23:27:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 23:27:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 23:27:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 23:27:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 23:27:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 23:27:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 23:27:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 23:27:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 23:27:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 23:27:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 23:27:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:27:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 23:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 23:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 23:28:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 23:28:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 23:28:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.43 s +[02-20 23:28:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 23:28:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:28:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/config.yaml +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 23:28:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera settles into a frontal close-up, focusing on the blue figure’s raised arms and the interplay between all three figures. The reflective black base mirrors their vibrant colors and dynamic poses. The modern glass building looms behind, reflecting the sculpture and the city skyline. Metal barricades remain visible at the periphery. The camera slowly zooms in on the blue figure’s silhouette against the glass facade, ending the video with a tight, detailed shot that emphasizes color, form, and reflection.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.json +[02-20 23:28:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 23:28:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.mp4 +[02-20 23:28:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 23:28:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 23:28:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 23:28:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 23:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 23:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 23:41:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 23:41:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 23:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 23:41:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..9e89cb511bcae2a3462308e007a553e1c9e6e159 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle shot of a colorful abstract sculpture featuring three stylized human figures — orange, yellow, and blue — mounted on a reflective black circular base. The camera begins a slow clockwise pan, revealing the glossy surfaces of the figures under bright daylight. Behind them, a modern glass building with large windows reflects the surrounding cityscape. Metal barricades line the edge of the plaza, defining the sculpture’s space.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..54918784aed57a29c3034fcbfcaba8d709b6b615 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abedbd2be0e17b2d0a288e619984fdbe08b6ba3fe08a8a4ed049d16cd66efdfa +size 1100129 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..1ee910cb97096ebe827ce627bd264d23326dbe1b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the clockwise pan, the camera focuses on the orange figure’s dynamic pose — arms outstretched, legs bent — while the yellow figure appears to lean forward beside it. The reflective black base mirrors the vivid colors above. In the background, the modern glass building remains visible, its facade catching glints of sunlight. Metal barricades continue to frame the scene, separating the sculpture from pedestrian walkways.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fe943e2623cefeadd91e503a086ff9995b587f7b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0e0b67dbde1d7e7dd6fe5a8d4b1ef89d32d53dc921dcc999a45d5c5da78cf3a +size 1173465 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..f08e54d4a61fc47962fcabd736bd5424665263fe --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera now reveals the side profile of the sculpture, emphasizing the blue figure’s towering stance with arms raised high. The yellow figure’s elongated limbs create a sense of motion, interacting visually with the orange figure. The reflective black base enhances depth by mirroring the forms. The modern glass building still dominates the backdrop, reflecting the sculpture and distant skyscrapers. Metal barricades remain visible along the plaza’s perimeter.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98a7ba045488160eaca94282f44c7bc85bc11cc7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:634921a613db6891c7cb46fc480d1228aa46c6388d648093ffb0f67224305c4f +size 1355858 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..d0ea6bc42941e208fed2d3a1809d4c98d8e17b23 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: As the camera continues its orbit, the rear view of the sculpture comes into focus. The blue figure’s back is now visible, its arms still raised, while the orange and yellow figures appear intertwined from this angle. The reflective black base captures distorted reflections of the figures and the sky. The modern glass building remains a constant presence, its windows reflecting the urban environment. Metal barricades are still present, guiding pedestrian flow around the artwork.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..355053cb6e6b9b4ddf8a0af93597c067d9bc9d92 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:780cf5ac61e1b24f680fcf19392cb1361e87c87bf2157ce4f8ad5744100c24c4 +size 1274219 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..38a6e4cb78f03e4c75e295fc383b314ebc4b821c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera now pans past the sculpture’s left flank, offering a new perspective where the yellow figure appears to stride forward, its limbs extending dramatically. The orange figure’s curved torso contrasts with the blue figure’s verticality. The reflective black base continues to mirror the scene. In the background, the modern glass building reflects both the sculpture and the bustling city beyond. Metal barricades remain visible, reinforcing the sculpture’s public placement.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d536cd70375f6a08eda372b4761db345a2fe8a3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a41b3ae896f3f1d33cb240c8abc54111194b1ac7c9142fd280455b6567b05df +size 1644612 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..bcf174974a0fc2152b0ebbc3758e08c954a6a1e3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: Continuing the clockwise movement, the camera captures the sculpture from a three-quarter rear angle. The blue figure’s raised arms dominate the upper frame, while the orange and yellow figures appear to support or react to its posture. The reflective black base catches the shifting light and colors. The modern glass building is still visible behind, its reflective surface adding layers to the composition. Metal barricades line the edge of the plaza, subtly guiding viewer attention toward the sculpture.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ab442209f9ec9b2b5526db16364a9b2ecea8e1d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9581ab73eb13d46f99eb925b97667a98150661545bfdea38c235bd9441fe9a40 +size 1530335 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..3e50e5cf831f34b89013bdb6e75d158bc409d002 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera now approaches the front again, but from a slightly lower angle, emphasizing the sculpture’s scale and the glossy textures of the orange, yellow, and blue figures. The reflective black base mirrors their forms with crisp clarity. The modern glass building frames the scene behind, its clean lines contrasting with the organic shapes of the sculpture. Metal barricades are still visible at the edges, maintaining spatial boundaries around the artwork.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c335f5f0b5a3eaaf3d77fa36de596781fc307a0a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fafa24c3263c5f7e71f0c83cd82b611f9fa21dac66d57a97373338a9861a9883 +size 1448413 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..387ee11e98cfa0310d2d6eb0a35356c7fc99f577 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera settles into a frontal close-up, focusing on the blue figure’s raised arms and the interplay between all three figures. The reflective black base mirrors their vibrant colors and dynamic poses. The modern glass building looms behind, reflecting the sculpture and the city skyline. Metal barricades remain visible at the periphery. The camera slowly zooms in on the blue figure’s silhouette against the glass facade, ending the video with a tight, detailed shot that emphasizes color, form, and reflection.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7829c4285859489ec993ec592591bed4f12eaa3e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba5c45962e3f1a6ee665430a37f8951cfc462f138193b75bc515281e65be1456 +size 1507099 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fe639c27efbf37e99265de4de83c4865a84ed14 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/console.log b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b6117bd174d1edc75426665d3f3482908ff86a44 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/console.log @@ -0,0 +1,22 @@ +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/config.yaml +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.json +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:09:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:09:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:09:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:09:42|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..623dfdad123f93f26b90679aeeabdee76ef86fb3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/debug.log @@ -0,0 +1,353 @@ +[02-19 22:55:35|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 22:55:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:55:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:55:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 22:55:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 22:55:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 22:55:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 22:55:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 22:55:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 22:55:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 22:55:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 22:55:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 22:55:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 22:55:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 22:55:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 22:55:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 22:55:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 22:55:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 22:55:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 22:55:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 22:55:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 22:55:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 22:55:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 22:55:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 22:55:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 22:55:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 22:55:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 22:55:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 22:55:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 22:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 22:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 22:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 22:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 22:55:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 22:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 22:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 22:55:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:55:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 22:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.17 s +[02-19 22:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 22:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 22:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 22:55:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.16 s +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.68 s +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.69 s +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.69 s +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 22:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:55:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.02 s +[02-19 22:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:56:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:56:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/config.yaml +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 22:56:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera returns to a near-frontal view of the distinctive pine tree, completing the full 360-degree orbit. The sun sits low on the horizon, bathing the entire scene in warm, golden light. The white curved pathway and background green shrubs and trees frame the tree symmetrically, concluding the sequence with a serene, balanced composition.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.json +[02-19 22:56:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:56:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.mp4 +[02-19 22:56:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 22:56:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 22:56:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 22:56:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:09:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:09:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:09:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:09:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:09:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:09:42|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d75ef20cf775874e2dd65297b8ae01d43667b15a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a static wide shot of a distinctive pine tree with layered, horizontally spreading branches, standing tall in the center of a sunlit lawn. To the left, a white curved pathway begins to appear, winding gently through the grass, with background green shrubs and trees softly framing the scene under a clear blue sky.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c0175dc1f546184bdd2ce514b919772729d5541 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7ca196871fb565b8f148b86b277dedb4a2721cd6ae112d072a953c3e008c36 +size 2239301 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..d7fe24fd350c6369e0ef45c7635c44b1b7db9d00 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera begins a slow clockwise pan around the tree. The layered branches become more visible, and sunlight catches the tips of the needles, creating subtle glints. The white curved pathway becomes more prominent, curving behind the tree, while background green shrubs and trees remain steady in the distance.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bc6f97d4a9a3f182caa9703219dadf3a9a6c9368 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b746623dd4711fc386fe8306389813a3c16e0e5437d50edcbf3ca32a310be3fe +size 1756723 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a16aba72fbaeb9a7b0b40df83e60589faa5d317c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: Continuing the clockwise movement, the camera reveals the left flank of the pine tree. The texture of its foliage is detailed, with individual needle clusters visible. The sun begins to peek through the upper branches, causing gentle lens flare, while the white curved pathway extends further into the background amid green shrubs and trees.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2de4497dedbce8ab21260be9ba26f481bccf5ec5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db4e8f0f46f3055d6cb7c5f180ff68ba9ac1ffcd154287fab90bc6afc63f96a1 +size 962273 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4b1af400b1889054b2827587ac1ea73d984320cc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: Midway through the orbit, the camera captures a side profile of the tree, emphasizing its sculpted, tiered structure. The white curved pathway now curves behind the tree, partially obscured by foliage. Background green shrubs and trees add depth, and the sun casts long shadows across the sunlit lawn.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b86ad48f1c010f25f65abd7dd68b5fbad3f7bf95 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9d808b330eaecfe8f2a2dfbfeb0d9b58445519c9cd3359b65272ebb319d37f9 +size 787063 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c58b44972b3d5724f986cc2d76377aeb9caa11ca --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera moves past the midpoint, now showing the rear-left quadrant of the tree. The layered branches appear denser here, and the sunlight illuminates the underside of the foliage. The white curved pathway emerges again on the far side, leading toward a distant treeline framed by green shrubs and trees.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e84ebc953fad2393f2f0675023f337849911d6a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8845db5cb7f9bfb288854f86851571b1440d5b666e7a43f6b0379832ce44a8d +size 1638590 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..928e22ca659466b6fc8c839144db53683369dd62 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: As the camera continues its clockwise arc, it reveals the right flank of the pine tree. The white curved pathway is now visible on the right side of the frame, snaking away into the background. Sunlight filters through the upper canopy, creating a halo effect around the tree’s crown, with background green shrubs and trees providing a verdant backdrop.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c751de7965d62b3c82a8dca9913b39ea5af9f9aa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f38096d42cf01da0959f7f3574ceab587fe2173348122964fb3d6239b284adbd +size 1854293 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..6bed5fd1689d415a44281f2b6adee07aabd6e9b8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: Nearing completion of the circle, the camera shows the front-right view of the tree. The layered branches are fully illuminated by the setting sun, casting dramatic shadows on the lawn. The white curved pathway re-enters the frame prominently on the right, leading toward a glimpse of a structure hidden among background green shrubs and trees.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40e9ce746090d58a20699482ef251fd341adec8b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26c88eb3d3ca36f5deb22e08ad6357bbec59c0b838cd9815471b954cee5f28ef +size 1751021 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..1b388f3c99d04e51222c540a9e011343bb58cf9c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera returns to a near-frontal view of the distinctive pine tree, completing the full 360-degree orbit. The sun sits low on the horizon, bathing the entire scene in warm, golden light. The white curved pathway and background green shrubs and trees frame the tree symmetrically, concluding the sequence with a serene, balanced composition.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9b0ff856b541ee977607ec4cf8a578f039119b8b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a691810e894860b33a14559312fcf3238a0110c5be88c7a858f3692ed741225 +size 1018084 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd42976f697832c09e031f32b5f657653869966d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/console.log b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/console.log new file mode 100644 index 0000000000000000000000000000000000000000..cca510e7ee41412dd730b0f475d83687b3eb1092 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/console.log @@ -0,0 +1,22 @@ +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:51:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:51:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/config.yaml +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.json +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:05:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:05:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1c6e4e8c4b2f45dcd44c1bb20d23fd8aa7c0715e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/debug.log @@ -0,0 +1,353 @@ +[02-20 01:51:04|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 01:51:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:51:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:51:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 01:51:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 01:51:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 01:51:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 01:51:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 01:51:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 01:51:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 01:51:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 01:51:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 01:51:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 01:51:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 01:51:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 01:51:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 01:51:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 01:51:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 01:51:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 01:51:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 01:51:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 01:51:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 01:51:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 01:51:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 01:51:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 01:51:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 01:51:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 01:51:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 01:51:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 01:51:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 01:51:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 01:51:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 01:51:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 01:51:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 01:51:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 01:51:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:51:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:51:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 01:51:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:51:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:51:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:51:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 01:51:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:51:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 01:51:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:51:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:51:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 01:51:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:51:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:51:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 01:51:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 01:51:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.13 s +[02-20 01:51:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:51:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:51:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/config.yaml +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 01:51:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its 360-degree clockwise rotation, returning to a near-identical starting position — facing the central white display with blue bags, the wooden shelves on the left, the red and white sailboat panel on the right, and the glass doors at the far end visible behind the display. The lighting, composition, and spatial relationships remain unchanged, creating a seamless loop that emphasizes the curated, professional atmosphere of the exhibition space.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.json +[02-20 01:51:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:51:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.mp4 +[02-20 01:51:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 01:51:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 01:51:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 01:51:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 02:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 02:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:05:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:05:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 02:05:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..7596ae85ce5329dcd4fad8f111c4861c2a2d8ee0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a wide shot centered on a modern, white, multi-tiered sculptural display unit illuminated from below, situated in the middle of a spacious room with light wood flooring. Behind it, a large wall banner displays the Chinese phrase '让生活有味' in elegant script, above which hangs a dark wooden sign with gold lettering '汇聚千年甲天下'. To the left, wooden shelves hold framed items. The camera begins a slow clockwise pan, revealing the left side of the room under bright recessed ceiling lights.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c7865de5cf37d590cea5caa25014d2233bb29a41 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c75f8a36e4ea60066284f03524462db29c3c8a677ed232e5db55f2f563f5b2b0 +size 636286 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..50740cc0c554f8eecc85333117eff794bb876a90 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the clockwise pan, the camera moves past the left wall’s wooden shelves and begins to reveal the right side of the room. The central white display remains in view, now showing two blue-and-white branded bags resting on its lower tier. The camera’s movement exposes the beginning of a red and white wall panel with a stylized sailboat graphic and partial Chinese text. Additional display panels with product information become visible along the right wall.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f037746faac1a6aa9509b19a821685d0406c918f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0830f517383139bcf0a1d06a03c715dbdcc5d98aa1b161aa6c117f23815ce64e +size 1471147 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..3310634ce334fcc7178ba8249fb4c450794bd873 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera now fully captures the red and white wall panel featuring the sailboat graphic and vertical Chinese characters '自强不息 永不满足'. Adjacent to it, more wall-mounted displays showcase products under warm accent lighting. The central white display is still partially visible in the foreground, anchoring the scene. The camera continues its smooth, steady rotation, revealing more of the right-side exhibits while maintaining consistent framing and brightness.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79a4d51307aecea39f5b9b739438f9494ad7e99f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5071596325467cf9db328b7b499746f7ccf637b0b279177b707ebd610acb714 +size 1242289 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b821ee44ee4cea25aabb35a15c449169ca390085 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: As the pan progresses, the camera moves further along the right wall, showcasing additional shelving units filled with neatly arranged products and promotional materials. The red and white sailboat panel recedes into the background. The camera begins to turn toward the far end of the room, where glass doors with metallic frames start to appear. The central white display is now seen from the side, with its illuminated base casting soft shadows on the wood floor.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c184b1deae9a9faab26ee0043f37a69d718489e9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f60e1e58001d53b4ce112d299f73ab0a45b696fc8ae69629fa0bee9742246674 +size 1174424 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..1a193e504aa72cb3158081d7eacfdca148279594 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera focuses on the far end of the hall, revealing the full view of the glass doors with metal frames, flanked by more banners and informational posters. The central white display is now visible in the distant background, framed between the glass doors and the right wall. The camera maintains its steady clockwise motion, capturing the depth of the space and the symmetrical arrangement of displays on both sides.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92a7128225f1c269d0d787a6f3afd128b76fc340 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dd78d0b1727524df1198a28bbee593b4bbfea478d1a7c7a5fecb2954c3231c2 +size 862717 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..ef248fa3dc8a0a44a95477a850753b3782a09f3e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera continues its rotation, now moving past the glass doors and beginning to reveal the left wall again. The central white display reappears more prominently in the frame, with the blue-and-white branded bags clearly visible on its lower tier. The wooden shelves on the left wall, previously seen at the start, come back into view, reinforcing the circular nature of the camera movement. Lighting remains consistent and bright throughout.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e2c8431784fa703578e2a9e395742d6d7fab720 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cef6b2ce3787f7f84004594d60bb84b86a5926bccdb1cd79e62697c8ad10846a +size 1106142 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..2af0325fdbe43351b930b431aa41acd91a01641c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera pans further left, bringing the wooden shelves with framed items fully back into view. The central white display is now directly ahead again, with the '让生活有味' banner and overhead sign clearly legible behind it. The red and white sailboat panel is now visible on the far right, completing the spatial loop. The camera’s motion remains smooth and deliberate, maintaining a consistent height and distance from the central display.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..878fbefd407b096b7d2cd9f45375162e82533b75 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55d8ca45fa1d93b7073a936e8591c61515fefac7643d9e5fa3d5a28bab397884 +size 1058015 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..e0a38204eeedc9d8f4d08b67d2ec312728030daf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its 360-degree clockwise rotation, returning to a near-identical starting position — facing the central white display with blue bags, the wooden shelves on the left, the red and white sailboat panel on the right, and the glass doors at the far end visible behind the display. The lighting, composition, and spatial relationships remain unchanged, creating a seamless loop that emphasizes the curated, professional atmosphere of the exhibition space.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa9208f5de066294b1fc8e88cb9cfc3c8ced3e6e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:205354ff43f93183055306c5957dfe7ba0fc4950b1e606e46731229b11dbd302 +size 709893 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..009a9906e085490e0f8aea63d6475f50e31cb683 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/console.log b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/console.log new file mode 100644 index 0000000000000000000000000000000000000000..3873a2dc1420abf41e5eec42920f097607f2dd93 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/console.log @@ -0,0 +1,22 @@ +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 05:52:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 05:52:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/config.yaml +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.json +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:06:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:06:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:06:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:06:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1ec34985e41fe1ab5637fb04fb8f64742d43aebb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/debug.log @@ -0,0 +1,353 @@ +[02-20 05:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 05:52:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 05:52:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 05:52:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 05:52:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 05:52:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 05:52:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 05:52:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 05:52:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 05:52:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 05:52:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 05:52:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 05:52:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 05:52:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 05:52:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 05:52:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 05:52:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 05:52:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 05:52:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 05:52:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 05:52:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 05:52:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 05:52:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 05:52:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 05:52:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 05:52:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 05:52:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 05:52:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 05:52:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 05:52:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 05:52:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 05:52:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:52:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 05:52:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:52:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 05:52:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 05:52:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 05:52:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 05:52:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 05:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 05:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 05:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 05:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.69 s +[02-20 05:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:53:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:53:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/config.yaml +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 05:53:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its orbit and returns to the initial framing. The black rolling cart is centered, with the white laundry basket on the middle shelf, the patterned pillows on top, and the blue step stool below. The beige sectional sofa, TV with colorful graphics, blue exercise ball, and floor lamp are all present in the background. The camera tilts up briefly toward the ceiling fan, then lowers to rest on the cart’s base and step stool, concluding the video with a stable, centered view.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.json +[02-20 05:53:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 05:53:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.mp4 +[02-20 05:53:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 05:53:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 05:53:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 05:53:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 06:06:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 06:06:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:06:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:06:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:06:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 06:06:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..41d7c914a34f04eb9d3b12625f9c5804e423b7fe --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle, wide shot focused on a black rolling cart with three tiers. The top shelf holds two patterned pillows with circular motifs in purple, beige, and brown. The middle shelf features a white laundry basket with blue handles, slightly tilted. The bottom shelf has a blue plastic step stool. In the background, a beige sectional sofa is visible, along with a large TV on a stand displaying colorful graphics. A blue exercise ball rests near the TV. The camera begins a slow clockwise orbit.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3f5fd6ca845e844c68c0f63e70b51b3bb710aa13 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0570b112ad59e537cf484dc4ee82764b32f989bfe4d91a5027b4b22ff7c0e7c2 +size 997236 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..c8f334b641d3ce021a23c13dad79bef47af9cd31 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing from the previous shot, the camera moves slightly right, revealing more of the beige sectional sofa and a floor lamp behind it. The white laundry basket remains prominent on the middle shelf. The patterned pillows on top are steady. The TV continues to display vivid animations. The blue step stool is visible beneath the basket. The carpet is beige, and ambient lighting fills the room.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ee23f20de2c02d3352c12e375f6349be3b2b527c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa1421e2aacbc823ae02531348b2093df74bc25796a1536b8b75f25e4933996c +size 1070886 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e87c4159b4ca3d03a59fb7bd940e3035113c0021 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera continues its clockwise rotation, now showing the side of the cart with the TV and blue exercise ball more centrally framed. The white laundry basket and blue step stool remain in view. The beige sectional sofa curves into the background. The patterned pillows on top are unchanged. The camera maintains a low angle, emphasizing the cart’s structure and contents.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..06a98ea2f81bdab415bb41d93e6f1673d777addc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d3131989fdb794e12fbf51f94deb6d5e969dc510b60ad2286d035ba804702f5 +size 1004435 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..673b414d5b1fe06f71b91eae7f96980b54d83d4c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: As the camera progresses halfway around the cart, it reveals a glass sliding door on the left side of the room. The TV screen remains active with colorful graphics. The beige sectional sofa extends across the background. The white laundry basket and blue step stool are consistently visible. The camera dips slightly downward, focusing momentarily on the bottom shelf before rising again.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67682dd97f70ada55b19821f04c814a1ad1b7f7f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d17f79836391b288accf74da163942fa4336bbb33fd92386f4cedd9a8679f8c4 +size 1345884 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..9a149b7a71a5162e9a96df2301901fcdb189009c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera continues its orbit, now showing the rear of the cart. The patterned pillows on top are still neatly placed. The white laundry basket is visible from the back angle. The beige sectional sofa and floor lamp are partially obscured but still present. The blue step stool remains on the bottom level. The TV’s colorful display is still visible to the right.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..35d1282b9dbe5880ca7b590953527779995a660e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6edf8d2c7cfb4bbb9f3496697ed7a08487369f4e1f2aa78171be08f92c52fbf7 +size 1914031 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..148888adb19377143f7b0487666894902ee2bb14 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera moves further around, approaching the front-left side of the cart again. The blue exercise ball near the TV becomes more prominent. The white laundry basket and blue step stool are clearly in view. The beige sectional sofa and floor lamp reappear more fully. The camera tilts up slightly to capture the full height of the cart and the patterned pillows on top.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d50e4a9570b99b5139ce3925d043f2fe7687d66 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1ae43968d1dc3084be7f36f4f4f877c43afdd4db3c9e7717dd6906b5d6c82a4 +size 2616030 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..ca142b78ded27a2679a2a2a6b2d2c1a893230f65 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: Nearing completion of the 360-degree orbit, the camera shows the cart from a nearly identical angle to the start. The white laundry basket, blue step stool, and patterned pillows are all in place. The TV continues to display colorful graphics. The beige sectional sofa and floor lamp are fully visible. The camera begins to tilt upward slightly, hinting at the ceiling fan above.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4bc8c23dfb914d922b115a250965db32c858165d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd678845e882ac42cb565d9de8eca8528f3475f95baccb8e8d56c59d322a49c9 +size 2280861 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..20a182388e51d69b17f9bb54179eea074198a08d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its orbit and returns to the initial framing. The black rolling cart is centered, with the white laundry basket on the middle shelf, the patterned pillows on top, and the blue step stool below. The beige sectional sofa, TV with colorful graphics, blue exercise ball, and floor lamp are all present in the background. The camera tilts up briefly toward the ceiling fan, then lowers to rest on the cart’s base and step stool, concluding the video with a stable, centered view.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a7f9a0389837931b4c0ed0a89443b2b81d04e6d9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0d800c4ee13dc65a395b11b4d1394adac5ff8606766e1c3db727f07f76aee53 +size 2223650 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d688ccd554472a224231271b0fed9ace473c4af5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/console.log b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/console.log new file mode 100644 index 0000000000000000000000000000000000000000..1f5381b91aa47d25a67831cec03556692772149c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/console.log @@ -0,0 +1,22 @@ +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 05:21:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 05:21:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:21:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 05:21:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/config.yaml +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.json +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 05:35:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:35:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:35:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:35:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9be335d9bb68ab7b312cbc0bd62fcf48fef1b56e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/debug.log @@ -0,0 +1,353 @@ +[02-20 05:21:30|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 05:21:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 05:21:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 05:21:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 05:21:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 05:21:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 05:21:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 05:21:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 05:21:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 05:21:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 05:21:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 05:21:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 05:21:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 05:21:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 05:21:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 05:21:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 05:21:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 05:21:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 05:21:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 05:21:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 05:21:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 05:21:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 05:21:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 05:21:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 05:21:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 05:21:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 05:21:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 05:21:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 05:21:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 05:21:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 05:21:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 05:21:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:21:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 05:21:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:21:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 05:21:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 05:21:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 05:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 05:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 05:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.04 s +[02-20 05:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 05:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 05:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 05:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:21:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.43 s +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.92 s +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.92 s +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.92 s +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 05:21:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:21:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 05:21:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 05:21:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 05:21:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 05:21:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 05:21:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 05:21:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 05:21:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 05:21:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 05:21:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 05:21:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:22:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 05:22:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 05:22:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.70 s +[02-20 05:22:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:22:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:22:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/config.yaml +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 05:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='The camera completes its 360-degree rotation, returning to the initial framing. The black office chair with the patterned pillow remains centered, surrounded by the familiar elements: the beige sectional sofa, the TV on its black stand, the red leather office chair, and the carpeted floor. The video ends with a stable, wide-angle shot mirroring the opening frame.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.json +[02-20 05:22:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 05:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.mp4 +[02-20 05:22:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 05:22:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 05:22:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 05:22:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 05:35:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 05:35:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:35:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:35:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:35:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 05:35:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0504f35f1a90d344c4fcf20084ee64714f4cfd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a low-angle shot focused on a black office chair with a colorful patterned pillow (purple, beige, brown circles) draped over its back. The camera begins a slow clockwise rotation, revealing a beige sectional sofa with a patterned blanket to the right and a TV displaying a cityscape on a black TV stand to the left. The carpeted floor and ambient lighting are visible.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e26b0ac02da518a6723da42fd785bee28d834e2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b37452efa2b52001dea7efb2ef4f0ca3cfb42663ba1e8b831221679407747df +size 1341245 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..73c60b0884309c1d2227448aa2db2d4fbd7a0d59 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its clockwise rotation, more of the living room comes into view: a red leather office chair positioned near a desk in the background, framed artwork on the walls, and a floor lamp standing near a doorway. The camera movement remains smooth and steady, keeping the office chair centered as the focal point.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9dd2e6af54ad52e46ef6aa55c4bab6c1e3a9c83f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d2a565bcec016ae24b2b452f9da2ca35574444bdc6956a1c78607e23e57f31f +size 1445447 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..830199c4443f2dddfe31af179f0a09de9dfe4013 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The rotation progresses further, now showing the opposite side of the room. The TV on its black stand remains visible on the left, while the beige sectional sofa extends across the right side of the frame. The patterned blanket on the sofa and the carpeted floor continue to provide texture and context to the environment.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5015058e2da7774f113828fb1da4142dbcb4df03 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:653c9b934756e1c389226fe8159ee5efd3ab1576c15aa01ad6bfedeb3dad2826 +size 1499643 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..feb42035bb8c9862923a9ab537289edc92083044 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The camera begins to lower its angle, dipping underneath the office chair. The underside of the seat becomes visible, revealing white warning labels affixed to the plastic base. The black five-star base with its casters is partially visible, resting on the beige carpeted floor.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1ee8c1bcd4ee205649e1a63968c69442b8543c30 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb4c7f10aa0b37af70efd199d8ea415b0c52a9847e480c51ef436b9159f55175 +size 1218602 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..796b47cd8c8600b16f6075b7726463941427b81e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The camera fully descends under the chair, offering a detailed view of the underside of the seat with warning labels clearly legible. The black five-star base is now fully in frame, with each caster making contact with the plush carpet. The camera lingers momentarily to emphasize structural details before beginning to rise again.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..89aefb2f2accd4c72e922ea60f346cf3d90c2951 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a61df9bbffd450bae37a511cd0b6018e61768f015d309a5b1c38e18027d7f6a +size 799481 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..1fe767b361bf6ace1dbe2d7e7c7274efd978cda8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"The camera tilts upward along the central column of the office chair, transitioning smoothly from the under-chair perspective back toward the room’s mid-level view. The black five-star base recedes from focus as the patterned pillow and upper chair structure come back into prominence.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48c4d6b067baa490f134f98e7905e13cf6ce56b6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc36e4e3da2b4b3b212c2ef9d9eb30f0810457a47e5ccdba03e96d1258b4b7b7 +size 803818 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..3ac2769d7b28ca8c05649d1dca0e7a995172d152 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Rising further, the camera resumes its clockwise orbit, now capturing the TV on its black stand displaying a cityscape, the red leather office chair in the background, and the beige sectional sofa with its patterned blanket. The floor lamp near the doorway and the ambient lighting remain consistent visual elements.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a063afb9501d53995db27719685a2ca831e3c17b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02c207578f0f0acd4c4b571b165be44e1caa461dc9f2aee1a70b17b0817d9220 +size 1269877 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..eee8f9169d7510a760b41c375deef2ff42e0991b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"The camera completes its 360-degree rotation, returning to the initial framing. The black office chair with the patterned pillow remains centered, surrounded by the familiar elements: the beige sectional sofa, the TV on its black stand, the red leather office chair, and the carpeted floor. The video ends with a stable, wide-angle shot mirroring the opening frame.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0bd7f27ad540db376627b9f0c909aadfd4e5f12d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a0fc5427e8a5766e434a5185607940a7d1968b01bf2f2fa094bd7ec6b948ef0 +size 1342279 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2708c002af97a8d72a00162ed55192f268ef874f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/console.log b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/console.log new file mode 100644 index 0000000000000000000000000000000000000000..aadbc50aa0a77259a4ec9893d35e822d2ce2aecc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/console.log @@ -0,0 +1,22 @@ +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:13:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:13:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/config.yaml +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.json +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:27:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:27:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:27:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:27:23|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..953c64a568e55c0c1eb48ed7845bb99ccb125264 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/debug.log @@ -0,0 +1,353 @@ +[02-20 03:13:17|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 03:13:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:13:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:13:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 03:13:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 03:13:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 03:13:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 03:13:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 03:13:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 03:13:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 03:13:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 03:13:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 03:13:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 03:13:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 03:13:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 03:13:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 03:13:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 03:13:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 03:13:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 03:13:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 03:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 03:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 03:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 03:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 03:13:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 03:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 03:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 03:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:13:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:13:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:13:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 03:13:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:13:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 03:13:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:13:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:13:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 03:13:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:13:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:13:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:13:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 03:13:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 03:13:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.72 s +[02-20 03:13:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:14:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:14:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/config.yaml +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 03:14:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera settles into a centered, balanced composition. The leaf-like finial at the top, the glowing frosted glass lampshades in the middle, and the brass decorative sconce arm at the base are all harmoniously framed, concluding the video with a complete and detailed view of the ornate fixture.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.json +[02-20 03:14:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:14:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.mp4 +[02-20 03:14:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 03:14:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 03:14:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 03:14:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 03:27:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 03:27:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:27:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:27:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 03:27:23|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..49d61b634fa2bcd75e5662af36cfafdbd5b49f2c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle close-up of an ornate brass wall sconce mounted against a textured mauve wall. The two frosted glass lampshades glow softly, shaped like inverted bells with flared rims. The leaf-like finial at the top of the central column is partially visible.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ddf43243c96ad55f5152c60e696712f05c88711 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:629ab673ee5ee37d4c88d8a248c3abd83a859314c16d9729f3d91cf80edf1d04 +size 482899 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..428a03f90f313af388aeeaf89bca6f6775e47a19 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera slowly tilts upward, bringing the leaf-like finial into full view — delicately sculpted and gleaming. The polished brass body with ridged detailing becomes more prominent, and the upper section of the brass decorative sconce arm begins to appear.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f32d530695e7415644416e5c4723e72bc65a61cd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a2896b9e750696fa0e45c153948fc039bf88fa96216a9cba75a340c54989bcd +size 461957 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..eb6db9a196fbf6e6b2bfb622cb560b65cd7ac17d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: Continuing the upward tilt, the camera reveals the full vertical structure of the sconce. The brass decorative sconce arm curves gracefully between the two lampshades, displaying intricate horizontal grooves and fluted edges. The frosted glass lampshades remain glowing and symmetrical.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0845ec438b0f6d2634704caabab3d3fd21ce7248 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:216d4b219f5954667085a64ff8db0ad274a98d98d0bb73255b0eca6c6ef93455 +size 429723 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..e0b8b80196d8732f3e6e1d09b97a741f07d401f0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera pans right, offering a side profile of the sconce. The brass decorative sconce arm is now fully visible, connecting the two lampshades with elegant curvature. The texture of the mauve wall is subtly illuminated by the soft glow from the frosted glass lampshades.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cd3a7f33bafac91660fa4021fe1f7e0a633b2736 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5aca0212650d6bb2ec9c7855d5c7abe2b43e6dfcf31491603e9e4ff3ef17f0be +size 451066 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c80b81b6be28d9840a6e3169c01081700b4a7648 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera begins to circle back left, repositioning to emphasize the symmetry of the fixture. The leaf-like finial at the top remains in frame, while the lower portion of the sconce — including the mounting plate and arm curvature — comes into focus.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..426574f5ef700e94f6428980561491ecdd87ba63 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:964430c8c8836d3e9d539ad88095faeac359ed93cbad8235c35ecbef5586c831 +size 543379 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..5ad6927716ce48976d9e875a2308c9de650bb7a9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera tilts downward slightly, reframing the lower half of the sconce. The robust mounting plate and the elegant curve of the brass decorative sconce arm are highlighted. The frosted glass lampshades continue to emit a warm, diffused light.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b3f4e714994abfb9ced5b09028db9fd267d9dfee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dbd4d0b61f381549be86f40fecba6feb0f1d9256283de408aad018b7032cad4 +size 644866 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..1962feb374850e9dd5f6d6bbbce497467bc4482f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera pulls back slightly while rotating clockwise, capturing both frosted glass lampshades in full profile. The leaf-like finial at the top and the brass decorative sconce arm anchoring the base are both clearly visible as the fixture glows warmly against the wall.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f38791f2b01f4c50e133d0273167553efbb8fdfd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a37b87e50dd715226b8ecfc59d8110c7a41053e30bf201bf5180f892ca6c21ae +size 792223 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..2e4ccb81af5bea4fda83289816c578a3578bcc1d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera settles into a centered, balanced composition. The leaf-like finial at the top, the glowing frosted glass lampshades in the middle, and the brass decorative sconce arm at the base are all harmoniously framed, concluding the video with a complete and detailed view of the ornate fixture.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..386f611b9630f104203c6ffb3895667ada1d9fcc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8e18390dc2cd42ba7805c4740907ea5492aef5e6b99acde8ee1f8874b9f3bf3 +size 864195 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fe49208fabbc7171f03674ca34df45e25b83823 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/console.log b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..cebf5cd49871060f59dd07b0b8e45e7e71d767cb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/console.log @@ -0,0 +1,22 @@ +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:14:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:14:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:14:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:14:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/config.yaml +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.json +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:29:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:29:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7b69700bcb9f15ff00f7e197c802914a4d4c29e5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/debug.log @@ -0,0 +1,353 @@ +[02-20 02:14:09|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 02:14:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:14:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:14:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 02:14:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 02:14:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 02:14:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 02:14:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 02:14:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 02:14:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 02:14:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 02:14:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 02:14:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 02:14:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 02:14:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 02:14:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 02:14:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 02:14:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 02:14:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 02:14:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 02:14:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 02:14:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 02:14:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 02:14:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 02:14:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 02:14:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 02:14:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 02:14:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 02:14:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 02:14:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:14:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:14:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:14:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 02:14:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:14:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 02:14:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:14:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:14:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 02:14:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:14:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 02:14:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 02:14:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 02:14:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 02:14:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:14:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 02:14:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 02:14:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 02:14:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:14:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:14:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 02:14:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 02:14:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.11 s +[02-20 02:14:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:14:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:14:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/config.yaml +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 02:14:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Completing the full rotation, the camera stabilizes into a medium shot that mirrors the opening frame but from a slightly adjusted angle. The ironing board with the laundry basket is centered, surrounded by the beige sofas, TV on black stand, blue exercise ball, black side table, floor lamp, and wall of framed pictures. The curtained window provides soft backlighting. All mandatory landmarks are clearly visible and spatially consistent, concluding the video with a calm, composed view of the entire living room.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.json +[02-20 02:14:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.mp4 +[02-20 02:14:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 02:14:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 02:14:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 02:14:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 02:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 02:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:29:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:29:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 02:29:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..5700455210b7595cea3d2877e89bc814732ce8ed --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle, wide shot centered on a white ironing board with a laundry basket resting on top, situated in the middle of a beige-carpeted living room. Beige sofas with patterned pillows are visible on either side. In the background, a TV on a black stand displays a nature scene. A blue exercise ball is partially visible next to a black side table. A floor lamp stands near one sofa, and a wall with framed pictures is seen behind the TV. A curtained window allows daylight to filter in from the right.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9f1fe8c061bd9d19994d19e694bef0ae152a2c6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18c66dec0770b115c3b85d549046746944a8d6b881f4b3c0a92e550477738650 +size 501387 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..465eb02db0391061dc24ab8b9946580b93349311 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera begins a slow clockwise rotation around the ironing board. As it turns, the view reveals more of the left sofa and the black side table next to the blue exercise ball. The TV screen remains visible, displaying changing outdoor scenery. The floor lamp’s base becomes more prominent, and the wall of framed pictures comes into clearer focus. The curtained window stays in frame on the right edge, maintaining consistent ambient lighting.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..74b680649a720642a8c8e86c73f600ead0c87002 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93832123cc120545fdcec97c014851ef40f26e5c4b00f5aac041198a1d841b19 +size 1098642 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..774f25c42c0a5e0c17cbc24de0c4d5e7c5adfb10 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: Continuing its rotation, the camera now faces the back wall directly. The framed pictures dominate the upper portion of the frame, while the TV on its black stand remains centered below them. The ironing board and laundry basket stay in the foreground, slightly off-center. The beige sofas curve around the space, and the blue exercise ball is now fully visible beside the black side table. The floor lamp casts a gentle glow on the adjacent sofa.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..69c10001a557da4e341726a188edee7226800263 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf8ef3dc7d7413c346d01fd9bc821f4902661885232c6a8c7ae5d052a0c43e4d +size 1149733 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..e0c16a5beb2a1ff47308a6b974ed905ada88a8fb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera rotates further, now showing the right-side sofa and the curtained window more prominently. Sunlight streams through the sheer curtains, illuminating the carpet. The ironing board remains central, but the perspective emphasizes the depth of the room. The TV screen flickers with motion, and the laundry basket atop the board appears slightly tilted due to the camera’s motion. The black side table and blue exercise ball are still visible in the mid-ground.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d9faed7b8c1d16d3adb2b921d8c2e2d845dacb3c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b91bbaf82d67d30d30beac5bd929670ee6aa41f3855bfa2aac577f31865ba1f4 +size 1452751 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..2edfad8935da1bb60919a2068231280b94eb6f57 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera lowers its angle significantly, dipping beneath the ironing board’s legs to capture the texture of the beige carpet and the metallic structure of the board’s support. The laundry basket hangs above, partially obscuring the view. Through the gaps between the legs, glimpses of the sofas, TV, and exercise ball can be seen. The floor lamp’s pole and base are visible on the left, while the curtained window glows softly in the background.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c34d22f40c9e79de8e77304b2773dd246517af75 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4a723a141320612288b32e4c3a79b0a9ea7fa60425f57ac09d9707bc562dff2 +size 1719144 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..fdbaa4bf250d536556d19a619283f08f557cfcf8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: Rising slightly, the camera continues its clockwise sweep, now capturing the laundry basket swaying gently as it passes over the ironing board’s surface. The view includes the front-facing sofa, the TV on its black stand, and the blue exercise ball beside the black side table. The wall of framed pictures remains visible behind the TV. The floor lamp’s shade is now in frame, casting a warm pool of light on the adjacent cushion.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c242487d122c0cd567aa1c6b638bab9be7d6fe4e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9d9422b4d1aba18322ff86bf8c15ff889c8843fbb3e9461b3d9f02d5744964a +size 1645229 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..35fe1452ff608e8297f9fb7e8f9d43aeb49da790 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera tilts upward briefly, revealing the ceiling and the top edges of the framed pictures, before descending again to resume its circular path. The ironing board remains the visual anchor, with the laundry basket appearing more stable now. The curtained window reappears on the right, and the arrangement of the beige sofas frames the scene symmetrically. The black side table and blue exercise ball are consistently positioned in the mid-ground.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..244c876be104b1656d3c4b6d7c7e8a4155525fc4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63941ab81ec2141de4696fec043f8a94bc3d979cdb7f50a866bee26fd032b63d +size 1389757 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..9e953c3bde2c7fb58846215b7a0bc42ae198c3a7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Completing the full rotation, the camera stabilizes into a medium shot that mirrors the opening frame but from a slightly adjusted angle. The ironing board with the laundry basket is centered, surrounded by the beige sofas, TV on black stand, blue exercise ball, black side table, floor lamp, and wall of framed pictures. The curtained window provides soft backlighting. All mandatory landmarks are clearly visible and spatially consistent, concluding the video with a calm, composed view of the entire living room.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1f31a4b5cc56948f9eedf43ddb9b96b41527bbef --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6f3fd964c1f3c7bcda963280520faf4ec5c7efec13580702c84c65befc8c7f4 +size 1234329 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f8d2e4f3489135054c7bfd221fb1b183abda781 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/console.log b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/console.log new file mode 100644 index 0000000000000000000000000000000000000000..0e7f66daacb4e873252076d060f45884d3b8bfb3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/console.log @@ -0,0 +1,22 @@ +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:02:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:02:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/config.yaml +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.json +[02-20 21:03:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:16:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:16:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:16:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:16:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e242e9466af968d3dfc4a9bc29b912db468f5ad7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/debug.log @@ -0,0 +1,353 @@ +[02-20 21:02:42|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 21:02:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:02:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:02:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 21:02:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 21:02:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 21:02:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 21:02:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 21:02:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 21:02:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 21:02:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 21:02:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 21:02:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 21:02:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 21:02:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 21:02:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 21:02:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 21:02:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 21:02:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 21:02:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 21:02:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 21:02:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 21:02:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 21:02:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 21:02:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 21:02:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 21:02:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 21:02:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 21:02:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 21:02:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 21:02:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 21:02:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 21:02:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 21:02:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 21:02:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 21:02:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 21:02:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:02:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:02:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 21:02:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:02:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:02:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:02:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 21:02:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:02:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 21:02:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:02:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:02:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 21:02:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:02:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-20 21:02:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 21:02:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 21:02:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 21:02:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:02:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:02:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:02:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:03:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 21:03:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 21:03:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.16 s +[02-20 21:03:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:03:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:03:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 21:03:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/config.yaml +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 21:03:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Finally, the camera completes its full 360-degree circular pan, returning to a centered frontal view of the curved wooden reception desk. All key elements — the blue countertop, glass-walled office, trash bins, EXIT sign, orange wall, seating area, and wooden ceiling panels — are simultaneously visible, concluding the video with a comprehensive overview of the modern reception area.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.json +[02-20 21:03:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:03:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.mp4 +[02-20 21:03:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 21:03:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 21:03:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 21:03:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 21:16:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 21:16:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:16:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:16:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:16:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 21:16:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..af34013daf8fb9cd06ff47bff293ae2363d9a5af --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of the curved wooden reception desk featuring a dark blue countertop. Above it, a white circular ceiling canopy with recessed lights and security cameras is visible. Behind the desk, a glass-walled office with light green tiled walls contains a black office chair and monitor. The camera begins a slow clockwise pan.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4da15399b831888385d6ec5de073aa8226cdfab2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43fdab7e42087a4cf81f1b933343580d381c305fe37cadafb1aa48229d01dbe2 +size 631901 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..7f520e4b272b68cdffab9a63c466ba03dac5eb29 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its clockwise movement, the left side of the reception desk comes into view, revealing an orange wall section adjacent to a seating area with tables and chairs near large windows. Wooden ceiling panels and exposed ductwork are visible overhead, adding texture to the industrial-modern aesthetic.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bce7983a11a9eae2da730793af3f778ab57e1888 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf88d612539a2cb1b0177c448506f9c26ed037cf3862d262148123ed911575f5 +size 1551154 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..b34b0a246e965fd58ee5bfea5407ebd179a475af --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera pans further right, showing more of the seating area and the edge of the curved desk. The dark speckled floor contrasts with the light wood of the desk and surrounding cabinetry. The lighting remains consistent, blending natural daylight and artificial ceiling fixtures.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15bf4838239d6b4fb0168168a68ee6d2fe0722ce --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:004900b91391fa515ce78e550b443d32554a5dd9b771d84658f43a15c83cf839 +size 1111005 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..0924fe85e5e993e35ca3c0b18adf3fad90fd3ba8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Continuing the circular motion, the camera reveals two labeled trash bins mounted on wooden paneling beside the reception desk — a gray bin labeled 'LANDFILL' and a blue bin labeled 'PLASTIC & METAL'. The bins are clearly legible and positioned for easy access.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cd2c1150e07b5d2f8171328688b38e926848c34 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baf9ad171ec2574a148fceef913894830d1e6515c57ac73a7e9ebb3024a2baf1 +size 1166646 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..21bcc74e1949af28cedc5459ea241e5e2c2b50f2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The camera pans past the trash bins, bringing into focus an illuminated red EXIT sign above a doorway in the background. The doorway leads to another corridor or room, partially visible through glass. The ceiling’s white canopy and lighting remain prominent overhead.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7da1467b9e2bd9bf20d3de1227e5169959a8350f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c3959dcf31949cbede2d384997997a85c7e7722be8eae31d5c6b595760d4a2c +size 799492 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..916864ec9817b3f65ce9b9376cea1f92e08fce2f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Moving further along the arc, the camera captures the rear portion of the curved reception desk, with the glass-walled office still visible behind it. The black office chair and monitor are again in frame, reinforcing the functional use of the space.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b01543bf431a048a69e5f5a72e88f85e08a3a23 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad29ca6b61496306f9c35682fd4a7ecd274a465cd86287e74baf69fffe512148 +size 906312 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..b02d8dea0f98dde05f93737f38460b7b8d3b97a2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"As the camera approaches the final quarter of its rotation, the orange wall section reappears on the left, framing the seating area once more. The combination of wood, glass, and metal materials creates a cohesive, contemporary look throughout the space.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8eb2ed6feee19679da83e6d70276d91b4a8ae4cc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37f0c9c188167967816171934f3e15afdab822c34ff6bff7d70fa08d562bb37a +size 673557 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..1186f906628b3efee3c65798eca06ff2f9bc1fdf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Finally, the camera completes its full 360-degree circular pan, returning to a centered frontal view of the curved wooden reception desk. All key elements — the blue countertop, glass-walled office, trash bins, EXIT sign, orange wall, seating area, and wooden ceiling panels — are simultaneously visible, concluding the video with a comprehensive overview of the modern reception area.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..90152604bdafaa5667dec6b3c93180737bf9f850 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ef761a39a0147d33f576a53735fa6fda3b3b0e9cafe061a260bf182ae5c5785 +size 720344 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0538dc4e241b8d6659364ea5027c04948ea14a12 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/console.log b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/console.log new file mode 100644 index 0000000000000000000000000000000000000000..9902902c05ed74ef258422b082b95e0281fa2d3c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/console.log @@ -0,0 +1,22 @@ +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:56:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:56:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/config.yaml +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.json +[02-20 11:57:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:10:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:10:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b71b0a6e7b5d0535aa9d3971d07aba34ee847a8c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/debug.log @@ -0,0 +1,353 @@ +[02-20 11:56:24|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 11:56:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:56:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 11:56:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 11:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 11:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 11:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 11:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 11:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 11:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 11:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 11:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 11:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 11:56:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 11:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 11:56:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 11:56:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 11:56:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 11:56:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 11:56:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 11:56:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 11:56:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 11:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 11:56:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 11:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 11:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 11:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:56:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:56:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 11:56:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:56:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 11:56:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:56:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:56:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 11:56:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:56:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 11:56:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 11:56:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 11:56:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 11:56:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:57:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 11:57:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 11:57:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.05 s +[02-20 11:57:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:57:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:57:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 11:57:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/config.yaml +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 11:57:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its arc, returning to the initial framing that includes the black curtain at the carousel entrance on the far left. The metallic baggage carousel, beige tiled wall, and ambient lighting remain consistent. The video ends with a stable, symmetrical composition mirroring the opening shot, closing the visual loop.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.json +[02-20 11:57:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.mp4 +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 11:57:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 11:57:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 11:57:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 12:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 12:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:10:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:10:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 12:10:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..7723933cf39cefcf9bf8501afafcaf4e5e57f821 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a slow left-to-right pan across an empty airport baggage claim area. The metallic baggage carousel dominates the foreground, featuring a polished stainless steel exterior and a black rubber conveyor belt. On the far left, a black curtain hangs at the carousel entrance, partially obscuring the dark interior behind it. Beige tiled walls extend into the background under recessed ceiling lights.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2bff38d0f2890e0a0953831df83bfa32eeba8572 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:209b09dc660137239ea1405d6eee2cd84a8c5d8255311bff61a3c7217deba98c +size 871026 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..4ffeca37a93aea721947400101d9d892b957615a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the pan, the camera moves slightly right, revealing more of the curved metallic baggage carousel. The black curtain remains visible on the left edge of the frame. The beige tiled wall continues along the back, and the terrazzo floor reflects ambient light. The camera maintains a smooth, steady motion, emphasizing the clean, modern architecture.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b73e49abc10f3f8b4704460c77acc4af5325a666 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aa4187bf482b5248273dc667de228e8980f378f4793106b7b6b6d36fdc65f83 +size 1003517 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..7157cfb3f5238f25c87741c6ec3380b0728e01f5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera continues its rightward pan, now showing the midsection of the metallic baggage carousel. The black curtain is no longer in view. The beige tiled wall stretches across the entire background, and the polished metal surface reflects the overhead lighting. The environment remains devoid of people or luggage, enhancing the sense of emptiness and sterility.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8c8027828670e385edb2a1234bb5d3b214723251 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e3078351dd7048c1f1a8d71d196dac02859c22c0aa18d933a5be79603e25712 +size 826436 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..69d7fb397617187723d44cae3f32234f6875a860 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: As the pan progresses, the right side of the metallic baggage carousel comes into full view. The beige tiled wall continues uninterrupted, and the terrazzo floor remains visible below. The camera motion is fluid and deliberate, capturing the sleek design of the carousel and the uniformity of the tiled surfaces.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d417669d798404d4b8bf08139da6624087d02f6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c32c2d38694e8ee2b9fc325c6d2c71c29ba52a0eea09b822609990f44c8f13ad +size 870512 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..bab9b585adbfc20a58db8697aed39aaa0bee3531 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera reaches the far right end of the carousel, where a blue directional sign becomes visible on the wall above. The metallic surface gleams under the lighting, and the beige tiled wall extends beyond the sign. The camera begins to slowly curve back toward the left, initiating the return sweep.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..698d85092f01a47e89f0ae91df9240572a158a73 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9ab1d7263869c8302629275c74557ccc6ee19a08e06e965fed12ede146bfe7 +size 883708 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..6319c7101ac26c30763de436ee80650d62c6e5a6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: Continuing the arc back to the left, the camera now shows the rear portion of the carousel from a slightly different angle. The blue sign remains in view for a moment before exiting the frame. The beige tiled wall and polished metal continue to dominate the scene, with no changes in lighting or occupancy.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0948c0399b0f59b15d643608b09e619d79ad52a0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e90e4a67f675bbaef1e369988f3373ceb596b21177d7629a5dd14e5a52cea523 +size 1115902 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..30b9bc8dbba1b522257006830ebc63bddc4a2fa0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera pans further left, returning to the central section of the metallic baggage carousel. The curved structure is fully visible, and the beige tiled wall forms a continuous backdrop. The terrazzo floor reflects the surroundings subtly. The smooth, looping motion reinforces the quiet, empty atmosphere of the space.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9026b4185283621d8a43438501ab0db47fa019c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f729d4d0d042bf1dcd17e2a7fb0e30ba46c99631d7102beb91e1cf7af387dca4 +size 1496394 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..4b9a8c32f668ba3caf6e76de5241dfdcde940fd1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its arc, returning to the initial framing that includes the black curtain at the carousel entrance on the far left. The metallic baggage carousel, beige tiled wall, and ambient lighting remain consistent. The video ends with a stable, symmetrical composition mirroring the opening shot, closing the visual loop.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..186ae78a09aa1fecac3485ea8f7a1606e507a0d4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce39e7ef9cfb4fd027606aac7f83f7bd2d7248c9a5a5d75efef2106206666ca3 +size 2100885 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b053dbfa3b4bd74233f534ad3d7e4d71c09503a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/console.log b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/console.log new file mode 100644 index 0000000000000000000000000000000000000000..9a29236829573bc99e1176783628ad9eedbffd80 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/console.log @@ -0,0 +1,22 @@ +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:59:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:59:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/config.yaml +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.json +[02-20 08:59:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 09:12:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 09:12:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 09:12:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 09:12:57|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..02ebd76d56ab0205e611aad7b27d4ac1fb6993bb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/debug.log @@ -0,0 +1,353 @@ +[02-20 08:58:48|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 08:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 08:58:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 08:58:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 08:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 08:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 08:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 08:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 08:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 08:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 08:58:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 08:58:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 08:58:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 08:58:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 08:58:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 08:59:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 08:59:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 08:59:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 08:59:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 08:59:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 08:59:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 08:59:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 08:59:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 08:59:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 08:59:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:59:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:59:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 08:59:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:59:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:59:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:59:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 08:59:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:59:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 08:59:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:59:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:59:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 08:59:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:59:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:59:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:59:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:59:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 08:59:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 08:59:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.21 s +[02-20 08:59:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:59:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:59:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/config.yaml +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 08:59:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Finally, the camera completes its 360-degree orbit, returning to the starting position but from a slightly lower angle, focusing on the ironing board’s legs and their shadows on the carpeted floor. The TV continues to display the city skyline, the beige sectional sofa frames the background, and the ceiling light fixture remains visible overhead — completing the seamless loop with all mandatory landmarks intact.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.json +[02-20 08:59:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.mp4 +[02-20 08:59:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 08:59:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 08:59:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 08:59:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 09:12:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 09:12:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 09:12:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 09:12:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 09:12:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 09:12:57|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8c2b1ba21ad6ca37beaca74a3cbb103ab3f15b38 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a low-angle wide shot of a living room featuring a beige sectional sofa with patterned pillows, a light gray carpeted floor, and a white ironing board centered in the frame, topped with a colorful circular-patterned pillow. In the background, a wall displays framed pictures, and to the left, a large TV screen shows a city skyline at dusk.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..36bbf289e16798826cb2da333b6dabe943aa3c42 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ec61829950c3dc994bc8fd3c90f4bafa3fd5357952cbc2780ce3ccfff257a36 +size 815093 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..ea8b388691e0f9551a9c23b4674b34d7f6c14089 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"The camera begins a slow clockwise rotation around the ironing board, revealing more of the beige sectional sofa’s curve and a side table beside it. The TV screen remains visible, displaying the glowing city skyline, while the carpeted floor extends beneath all furniture, showing subtle texture under ambient lighting.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..477383986454c94af2e60ce826cd950268b154af --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d99b59a8f8288d12d49d94c3a280d6ca38cdf6db79c60b56634454828926dcc +size 1332268 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..ae1ac55b9a2b1cdab387d054c3421c1edebad972 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"As the rotation continues, the camera captures a tall floor lamp near a window with sheer white curtains, allowing soft daylight to filter in. The ironing board remains the focal point, its white metal legs casting faint shadows on the plush carpeted floor, while the wall of framed pictures stays visible behind the sofa.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3a9c541f1dff21c72d5f574a75656b12e54e1d1c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:196f466e19224d90d440e3215734d8fd3b48050b8bbae186596830bff89a3c85 +size 1112161 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..fd4efce8ef80290d04585f5ea7f548731fa1a799 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The camera tilts slightly upward, revealing the ceiling light fixture — a modern design with frosted glass shades — positioned directly above the ironing board. The underside of the ironing board mesh becomes visible, showing its crisscross white fabric structure, while the TV screen reflects ambient light from the room.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34b7df9e4202ff7b8ccd44288d320fba34a36ee1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91ebf81f3031895360434b8eb1c99037993a772c3681715947088a465ae0a3e5 +size 1956453 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..1c773f1a29afaa6991a2ef346d87f394771dafca --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Continuing the clockwise rotation, the camera pans past the TV, capturing its full display of the city skyline with illuminated skyscrapers. The beige sectional sofa curves into the frame on the right, with its patterned pillows and blankets undisturbed, while the carpeted floor stretches continuously beneath all objects.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7c17aa216cb061a5f599edebae900410c90bb8b3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e399b2818b5a40cc36dbbb256bfab4f8c106805920dd1fa31d2a5b3ae93e5242 +size 1311717 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..849ef5125dd8733eb517a5cf8d68f09c5e5a75b3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"The camera moves further around the ironing board, now capturing the back of the sofa and the wall of framed pictures from a different angle. The ironing board’s legs are clearly visible, casting defined shadows on the carpeted floor, while the TV screen glows softly in the background.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df5768e9b254cf3437d9f08cf29e563ed7c8109d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb90bc5761d5cabf55a81c56958adc7d670c63e6da3b715645b91c2bea896ce3 +size 1434037 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..57118553d193e20dc0c262a07a9a6646f04e5596 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"As the camera approaches the final quarter of its rotation, it lowers slightly, emphasizing the texture of the carpeted floor and the base of the ironing board’s legs. The TV still displays the city skyline, and the beige sectional sofa frames the right side of the shot, maintaining visual continuity with earlier frames.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79c18f338d01bfc382e080cf142d5c800d29eefe --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5c4e734f6eabce53bdb30f936ea1cf8726178cf6abcbd0b46b6553c4e51d055 +size 1066777 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..462f279015effac503ddfd42e4f657220d0f4d1f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Finally, the camera completes its 360-degree orbit, returning to the starting position but from a slightly lower angle, focusing on the ironing board’s legs and their shadows on the carpeted floor. The TV continues to display the city skyline, the beige sectional sofa frames the background, and the ceiling light fixture remains visible overhead — completing the seamless loop with all mandatory landmarks intact.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1739ee94aac741e03a53fe56511751bfb0b7a107 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da1d829e5bbc9868a5a79bfe03ff78b62b48fd0e3027b37f07438ce97827ff60 +size 1208627 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1589da7ad03c3e2b4de6f4ce779678497ca9d4f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/console.log b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..d930a42a64e11b9e83fdd53ae1b2f681ad866f6d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/console.log @@ -0,0 +1,22 @@ +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 15:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 15:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/config.yaml +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.json +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:01:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:01:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:01:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:01:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..468fb83a9a206f368dd292bcd95be571494ab233 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/debug.log @@ -0,0 +1,353 @@ +[02-20 15:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 15:47:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 15:47:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 15:47:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 15:47:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 15:47:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 15:47:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 15:47:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 15:47:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 15:47:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 15:47:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 15:47:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 15:47:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 15:47:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 15:47:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 15:47:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 15:47:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 15:47:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 15:47:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 15:47:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 15:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 15:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 15:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 15:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 15:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 15:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 15:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 15:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 15:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 15:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 15:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 15:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 15:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 15:47:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 15:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 15:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 15:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 15:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 15:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 15:47:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 15:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 15:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:47:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 15:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 15:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:47:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 15:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 15:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:47:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 15:47:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 15:47:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.97 s +[02-20 15:47:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:47:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:47:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/config.yaml +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 15:47:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera holds steady on the uppermost branches, silhouetted against the night sky. Faint background building windows remain visible in the distance, twinkling subtly. The scene is calm and still, concluding the slow, immersive journey through the illuminated tree canopy under urban night lighting.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.json +[02-20 15:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.mp4 +[02-20 15:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 15:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 15:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 15:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 16:01:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 16:01:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:01:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:01:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:01:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 16:01:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3d1c027a1c9188dabb3240bc32ebf27bb435f65a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle nighttime shot centered on a large tree’s trunk, which fills the lower third of the frame. Illuminated leaves glow softly near the top, while background building windows emit a gentle yellow light in the distance. The camera begins a slow upward tilt, initiating the journey through the canopy.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77a5420a604f2f8143566c1b8b6eb7b6df149abf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2971261878bc5844c43ff46663362b5732f1ce3602ea68a30675b596a33f908 +size 668823 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..417b0e9d14c488643293961aa3a77121fb84ac0b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the upward tilt, the camera reveals more of the tree’s dense foliage. Illuminated leaves become more prominent, catching stray light from an unseen source. The background building windows remain visible but slightly obscured by branches. The tree trunk continues to anchor the left side of the frame.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e425efc1e5f29d766b0dab5604a3d1a9883ea41c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65943628c79275662bdc9e025e454a45259ff5c8d18b42808eb1d1c5b9634519 +size 769112 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..913ffc8a259c138b4cab9f5c535f48895d3e23c3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera ascends further, bringing the upper canopy into view. A sudden burst of streetlight glare appears in the upper right, washing out part of the frame with intense white light. Leaves near the glare shimmer with high contrast, their veins and edges sharply defined.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..35b7307225a494a6cd128a48c437f98f95d6d652 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5340aa34278223810be6846752d2e0e9fa77705bd2a448479a26466e800708a +size 898513 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..2155f74be77d45c7463b5a6af0945c8a6dd1a930 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The streetlight glare reaches its peak intensity, dominating the center of the frame and temporarily obscuring details of the leaves. The tree trunk is still partially visible on the lower left. Background building windows flicker briefly through gaps in the foliage, maintaining spatial context.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b033c3e080ee7b0e4440a565e19304db0c44bc1c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:405982b3980d8dca8ca835a66f4cac67d351e14d6d8830a7a9863301d60d588d +size 601551 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..0920d7b09b48ea72a36953bfc4fdb48cbbd1db03 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: As the camera moves past the glare, the brightness diminishes, revealing detailed illuminated leaves once again. The camera begins a subtle pan to the right, following the curve of a major branch. The tree trunk recedes slightly from view but remains present at the edge.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d4721d0fd8575d851ca5dbf6db30a4289afdecee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76c7199f9242a5d0e2c4ca32bc48dd7b133007b4b976321cf603d57085834f1f +size 1407701 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..49610d084fc6c7db662aff0be999f2fa748fa4f1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The pan continues, shifting focus toward the right side of the canopy. Illuminated leaves dance gently in the breeze, their textures and shapes clearly visible under ambient lighting. Background building windows peek through the gaps, reinforcing the urban environment.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7f829317ea9dfc8c75676b11dbd44a02e33daad1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0a330b97e6cccbe06da215380947fd998499603c954bdcc9c04f7540d5b1d12 +size 2473022 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..8505e9e719fc2a34041ba069e744b9adfe628c00 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera completes its upward arc, now focusing on the highest branches. The tree trunk is no longer visible, replaced by thin, silhouetted limbs against the dark sky. Illuminated leaves are sparse but still present, glowing softly in pockets of light. The streetlight glare has vanished entirely.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57c0807c0e327ae975edd0e88a788e8692b4c261 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59738d1e953117c06c9b400efa49a6923b66368c7b4c0f5e869112d709a5453f +size 1564323 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..d671388dfe6544ea351ded95b47bf1d10c033c2a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera holds steady on the uppermost branches, silhouetted against the night sky. Faint background building windows remain visible in the distance, twinkling subtly. The scene is calm and still, concluding the slow, immersive journey through the illuminated tree canopy under urban night lighting.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a492c06a7178b51aec1dbd87e75c5980a7da33e7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d80ab1a80cfd64fa01dcba716e78187d6d4d7ffdd8f0ec7a651b58a7aa5eafc +size 1179374 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f55811a31dad8afa4732880833ddfb4f30b02b15 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/console.log b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b2f510ad729224710a901002fc119d3135a59c1a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/console.log @@ -0,0 +1,22 @@ +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:43:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:43:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/config.yaml +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.json +[02-19 22:44:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:57:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:57:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:57:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:57:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2d45b1704aa81062a54920d1ab90aadefb39a83e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/debug.log @@ -0,0 +1,353 @@ +[02-19 22:43:31|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 22:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 22:43:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 22:43:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 22:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 22:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 22:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 22:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 22:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 22:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 22:43:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 22:43:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 22:43:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 22:43:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 22:43:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 22:43:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 22:43:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 22:43:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 22:43:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 22:43:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 22:43:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 22:43:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 22:43:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 22:43:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 22:43:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 22:43:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 22:43:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 22:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 22:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 22:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 22:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 22:43:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 22:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 22:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 22:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:43:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:43:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:43:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 22:43:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:43:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 22:43:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:43:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:43:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 22:43:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:43:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.56 s +[02-19 22:43:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 22:43:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 22:43:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 22:43:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:43:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.64 s +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 1.14 s +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 1.15 s +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 1.15 s +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:43:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:43:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:44:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 22:44:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 22:44:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.15 s +[02-19 22:44:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:44:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:44:20|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/config.yaml +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 22:44:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its rotation, ending with a frontal view of the black office chair with the patterned pillow. It tilts down briefly to emphasize the base and casters, then rises to frame the chair’s backrest against the backdrop of the living room. The yellow leather sofa, throw blanket, framed pictures, floor lamp, TV on black stand, and red gaming chair are all integrated into the final shot, creating a harmonious and complete visual summary of the space.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.json +[02-19 22:44:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:44:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.mp4 +[02-19 22:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 22:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 22:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 22:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 22:57:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 22:57:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:57:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:57:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:57:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 22:57:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0394650c531306e033ceded99983021f91c260d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a wide-angle shot of a living room centered on a black office chair with a patterned pillow. The carpeted floor is beige. Behind the chair, a yellow leather sofa is visible with a patterned throw blanket. Wall-mounted framed pictures decorate the wall above the sofa. To the left, a TV on a black stand displays a cityscape. A floor lamp stands near the sofa. The camera begins a slow clockwise rotation.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fbd2a8b0e93a31ed08ad582e4a9c2c866440b62b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:556cc60c68bb438b25f6d4ce03f288c52fcc13f64b2f202056599f32c230299d +size 1243071 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..1ac1fca4f0a13d7d20efc7fa625c1541ed6a1f57 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the rotation, the camera reveals more of the yellow leather sofa and the throw blanket’s leafy design. The TV on the black stand remains in view, showing the cityscape. The lower portion of the TV stand is visible, including a white circular object beside it. The black office chair with the patterned pillow stays centered as the camera moves, maintaining focus on its structure and position.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5981c6c02aa6729b1c8e35c81e4f333f88f48b9c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:170d44dca0b59912f5038201997d9665ac61777caa7409e1c36e2b65517a52f9 +size 1032588 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..249e8e4cf994092a0b5e5cafd7dceec760cf8d00 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera angle shifts slightly lower, offering a clearer view of the base of the office chair with its five black casters on the carpeted floor. The red gaming chair becomes visible in the background near the TV area. The wall-mounted framed pictures are still present above the sofa. The patterned pillow on the office chair features vibrant floral designs. The camera continues its smooth clockwise motion.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..26e56a802d86725c64114582aa4731cd17e76c7a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4c95c1f08be94239d274b121dc92a2370c3df4a86b996d50b0b3dd684aeb67a +size 828406 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..05473d6203bbe16d96e905b9d565dfd2828a3a8f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: Now viewing the scene from the side, the camera captures the full view of the black office chair with the pillow. The yellow leather sofa curves into the frame, displaying its plush cushions and throw blanket. The floor lamp near the sofa casts a warm glow. The TV continues to display the cityscape, anchored by the black TV stand. The red gaming chair is momentarily prominent before the camera rotates further.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2bc10edca1cc716c6fe28ed6728499f898ac3d93 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:909cb0c3ad9dea5c785d2469cbf3057bfa6983ad694720a7a83f1d360e10909b +size 910448 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..29d1305fa61bc23934a7187dd670ff1edf0a1d96 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera tilts downward, focusing on the base of the office chair and its casters rolling over the beige carpet. The black mesh back and armrests of the chair are visible above. The patterned pillow remains securely in place. The yellow leather sofa and wall-mounted framed pictures are still in the background, though partially obscured by the chair’s structure. The rotation continues steadily.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48b678419f4e968eb97fbac0a7bd182641933202 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a4bb1b7f9147b2a123c1bd7a3f3e32212803728cacf6f7bc65ca50e6072cc00 +size 931425 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9d9c219d546b231840dd5c59551d5ba134e8aa92 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: As the camera rises again, it frames the upper portion of the black office chair, highlighting the detailed floral patterns on the pillow. The TV on the black stand is visible to the left, still displaying the cityscape. The lower portion of the TV stand and the white circular object beside it come into view once more. The red gaming chair reappears briefly in the background. The camera maintains its clockwise path.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..636747d63844915d4b7946dbff18b60dc2e722e6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a0935fb69963537710185a7eae606c6018f6f7928544707133602b7f9931dc4 +size 1171164 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..562757817fedf88292d0058fff69f30ab4b56dc0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The rotation brings the camera back toward the front of the room, capturing the central black office chair from a slightly elevated angle. The yellow leather sofa, patterned throw blanket, and wall-mounted framed pictures are all visible again. The floor lamp near the sofa adds depth to the composition. The TV continues to play the cityscape, and the black TV stand grounds the visual field. The camera movement remains fluid and continuous.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc66fd5e9f34d2797789bca8bf8fb437f2dbc295 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3581b471e0195a17e985e9c7e2f8ff91155ca3de7448f1f31f7e2b0b741779f +size 1041254 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..63eb4b49b33572babb04be4171ab80d4457a60d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its rotation, ending with a frontal view of the black office chair with the patterned pillow. It tilts down briefly to emphasize the base and casters, then rises to frame the chair’s backrest against the backdrop of the living room. The yellow leather sofa, throw blanket, framed pictures, floor lamp, TV on black stand, and red gaming chair are all integrated into the final shot, creating a harmonious and complete visual summary of the space.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..36869a098b0f6ba72762a305291a30bfb4a74c62 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb46b926381d18e53df4aae322db9a843ef30250b7463bbb7c4cb2adaa6ed35e +size 1150380 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de71328156fe8861df18c159e81a01eca2ad327b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/console.log b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/console.log new file mode 100644 index 0000000000000000000000000000000000000000..9b2d77d9b8fe493d01ac263f79e9c6932fe3f34e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/console.log @@ -0,0 +1,22 @@ +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:55:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:55:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/config.yaml +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.json +[02-19 23:55:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:09:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:09:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:09:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:09:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8c91fa140c42526ff34b2ce6d2bf2bd312350de2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/debug.log @@ -0,0 +1,353 @@ +[02-19 23:54:51|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 23:54:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:54:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:55:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 23:55:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 23:55:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 23:55:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 23:55:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 23:55:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 23:55:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 23:55:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 23:55:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 23:55:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 23:55:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 23:55:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 23:55:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 23:55:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 23:55:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 23:55:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 23:55:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 23:55:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 23:55:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 23:55:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 23:55:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 23:55:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:55:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:55:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 23:55:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:55:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:55:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:55:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 23:55:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:55:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 23:55:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:55:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:55:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 23:55:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.16 s +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.80 s +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.80 s +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.80 s +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:55:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 23:55:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 23:55:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.07 s +[02-19 23:55:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:55:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:55:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/config.yaml +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 23:55:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Finally, the camera completes its full 360-degree circle, returning to a frontal, slightly low-angle view of the black and white abstract sculpture. The tall yellow-and-black striped pole stands prominently on the left, balancing the composition. The red vehicle, modern glass building, and autumn trees are all visible in their original positions, creating a seamless visual loop. The gravel path curves gently behind the sculpture, tying together the elements of nature, art, and architecture in a tranquil, cinematic finale.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.json +[02-19 23:55:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:55:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.mp4 +[02-19 23:55:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 23:55:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 23:55:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 23:55:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 00:09:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 00:09:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:09:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:09:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:09:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 00:09:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c2316be127d4e6a61ba2b2c4af26c1bd0db93b28 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a low-angle, wide shot of a black and white abstract sculpture on a green lawn. Trees with autumn foliage frame the background under a cloudy sky. A red vehicle is faintly visible parked in the distance near a modern building with a glass facade. The camera starts a slow clockwise circle around the sculpture, revealing its layered, flowing form. A gravel path curves behind the sculpture, partially visible at the edge of the frame.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e143537836efea1b081791bae50ab6a6615f4bf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5910ccfb0de0c5562b33ad9f0514cc003ec89cd676b7800ef7f44c3158fa48c +size 2034357 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..44f9a2fdc31731d8015feb463982ce042db0bcff --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its circular motion, the sculpture’s intricate cutouts become more apparent, casting shifting shadows on the grass. The tall yellow-and-black striped pole enters the frame on the left, standing upright beside the lawn. The red vehicle remains in the background, now slightly more defined as the camera angle shifts. The modern building’s glass facade reflects the overcast sky, adding texture to the distant background.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d5322034d6f6686102ac7f2f199f20b4a2550264 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2898b972c436151142ae98a158b1d8ab5a9f56ec764bb0a873bd2aaf45d47538 +size 1536297 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4ccd141f736066a9a386882cf09ead24ba81c00d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera moves further along its arc, now capturing the sculpture from its side profile. The depth of its layered design is emphasized, with negative space forming elegant voids. The gravel path behind the sculpture becomes more prominent, curving gently toward the modern building. The autumn trees sway subtly, their colors rich against the gray sky. The red vehicle is still visible but partially obscured by foliage.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d173ae8ece4935cc913bdd97db3225ffd8797f4d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97d61cc6ad112af8d0f9e2beaaa79f4e5a966c4d3128306e05e2f273fe0e3a81 +size 1384508 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..44614c1e058cfa59dffe187e676c4a77e1e7b6b0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Midway through the rotation, the camera lowers slightly, offering a closer view of the sculpture’s base, which is set into a patch of exposed earth or gravel. The tall yellow-and-black striped pole exits the frame as the camera continues its orbit. The modern building’s glass panels catch ambient light, creating soft reflections. The red vehicle remains stationary in the distance, serving as a fixed point of reference.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..27efb779cb859b351ea9724626ea4af3330fdaad --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3fbc25140822169d263b34990017160f9830053990adc4ef67381d29b70e5c6 +size 1265975 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..be0754f0a3c1bb48f56acc23b1ed0679d8d04986 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The camera now captures the sculpture from its rear-left angle, showcasing the curvature of its outermost layer. The gravel path winds visibly behind it, leading toward the modern building. Autumn trees continue to frame the scene, their leaves rustling in the breeze. The red vehicle is momentarily hidden by tree branches before reappearing as the camera progresses.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87708eff71f658b3c2ce0046790a665cc94e8fca --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4ffd86646b7e3a33d7ae7462224f3077057a72ef551f5b5a1352c111bd2d7ac +size 2437053 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..b5e06cd1b7a3ed18f5e18b6d37452da4c7d6e8d6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"As the camera nears the three-quarter mark of its circle, the sculpture’s front-right face comes into view, highlighting its symmetrical yet organic form. The tall yellow-and-black striped pole reenters the frame on the far left, anchoring the composition. The modern building’s glass facade glints softly, and the red vehicle is clearly visible again, parked near the treeline. The gravel path remains a consistent visual guide behind the sculpture.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c15aecb5ae5f9cae1f50d74161cd1606c5e33903 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:256a6f686c07a30a54d6d138e6e3327a3b6a8c0ef74fddaa7472954d28e1514c +size 1919626 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..11dba339f2f08fdc960f09739627b26380808431 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"The camera continues its smooth orbit, now approaching the final quarter of the circle. The sculpture’s layered ribbons appear to ripple with motion due to the changing perspective. The base, embedded in earth, grounds the piece visually. The autumn trees provide a warm-toned backdrop against the cool gray sky. The red vehicle, modern building, and gravel path remain consistently placed, reinforcing spatial continuity.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..44a9cb6ac351179153dae6cf6746d5d4f54281d4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36b49ad702a8fac45b1f7647f04636a40b42223bbaa8367bd44a6ce2a2b362b5 +size 1397974 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..736c51379a1cb4bc925743f07f28965e98c27a01 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Finally, the camera completes its full 360-degree circle, returning to a frontal, slightly low-angle view of the black and white abstract sculpture. The tall yellow-and-black striped pole stands prominently on the left, balancing the composition. The red vehicle, modern glass building, and autumn trees are all visible in their original positions, creating a seamless visual loop. The gravel path curves gently behind the sculpture, tying together the elements of nature, art, and architecture in a tranquil, cinematic finale.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b0458044752f9b9ded411916f80d889b081e6e2e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69107a2a3fbf3253b7d82c4227fc33c124f6fd1b68aefa1c4b96b286aee43833 +size 1695219 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c4909f1a608d67a69b55253878fc4c6d99958ad --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/console.log b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/console.log new file mode 100644 index 0000000000000000000000000000000000000000..042055905b8e1396604fdd4654439ec51986a807 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/console.log @@ -0,0 +1,22 @@ +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:14:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:14:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/config.yaml +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.json +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 19:28:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 19:28:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 19:28:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 19:28:49|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2b5c8f2d789a1ded26382e3e40a432bb20440b37 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/debug.log @@ -0,0 +1,353 @@ +[02-20 19:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 19:14:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:14:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:14:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 19:14:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 19:14:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 19:14:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 19:14:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 19:14:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 19:14:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 19:14:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 19:14:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 19:14:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 19:14:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 19:14:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 19:14:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 19:14:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 19:14:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 19:14:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 19:14:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 19:14:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 19:14:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 19:14:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 19:14:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 19:14:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 19:14:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 19:14:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:14:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:14:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 19:14:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:14:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:14:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:14:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 19:14:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:14:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 19:14:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:14:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:14:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:14:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:14:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:14:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 19:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 19:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.31 s +[02-20 19:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:15:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:15:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/config.yaml +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 19:15:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera holds steady on a wide, balanced shot of the entire picnic area — stone table, stump seats, willow trees, and the purple-painted path curving gracefully on either side. The distant park benches are visible in the background, completing the tranquil park scene. The lighting remains bright and natural, with shadows gently shifting as the video concludes.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.json +[02-20 19:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 19:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.mp4 +[02-20 19:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 19:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 19:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 19:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 19:28:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 19:28:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 19:28:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 19:28:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 19:28:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 19:28:49|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..727e57a71836ccdd2472d313945916bd1a5adb5a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a wide, sunlit view of a park featuring a stone picnic table surrounded by tree stump seats. Willow trees with long, hanging branches encircle the area. A purple-painted path curves to the left in the background, with distant park benches visible along its edge. Shadows from the trees dapple the ground beneath the table.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..437ed16f5403d1a7761d4ae5dd1384bd99e163dd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:398a9c03091dc6724d00bbb54165e65376e92718fec1e903482108b92523298c +size 1772619 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..bead46a788b4c99e8a309528dfe077d59046786d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera begins a slow clockwise pan, shifting focus slightly to reveal more of the purple path as it bends to the right. The stone picnic table remains central, but the perspective now includes more of the grassy bank and additional willow trees to the right. Distant park benches are still visible, now appearing slightly closer due to the camera’s movement.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da156b0d39a12811f0c898d806274f7b0f6b1b99 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2835a1bfb0dc18389faa9abff00e045be645cd1c03ae90e56cabde6c1914f9c +size 1677580 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..6a8612c4f76077e8d1a864a449578b063e0792aa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: Continuing the pan, the camera reveals the rear side of the picnic table setup. The purple path now stretches farther into the background, winding past a gentle slope covered in grass and small shrubs. More willow trees come into view, their branches swaying lightly. The distant park benches are partially obscured by foliage but still discernible.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1390253aa309da806ae01bf4605b1a41ebbf4d5d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8897853595a8e018eedacf68fed375234336da85261d1e87c02682ed291efad3 +size 1232778 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b30b1b7458648cf36411a3e97e1994d7a9a7ccfc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera continues its clockwise motion, now capturing the right flank of the picnic area. The purple path curves sharply to the right, leading toward a cluster of trees. The stone table and stump seats are seen from a side angle, with sunlight glinting off the stone surface. The distant park benches are now more clearly visible, with one person sitting on a bench.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..96396f2ff17bd4197458005cd1d8feb78c2e5fcc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bad50f4b721f32b9212d703c3d6aba2f453917ea96064849c858ca8c74bea61 +size 1391427 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e1546e2c944a5f6952d922a67aff6da3a24e86ae --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: As the pan progresses, the camera tilts slightly upward, bringing the canopy of willow leaves and the blue sky into greater prominence. The stone picnic table and stump seats are now viewed from a higher angle, emphasizing their placement within the natural setting. The purple path continues to wind through the background, and the distant park benches remain visible at the far end of the curve.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95f04f9422fd7bf0c15ef066cac33c29465c0d71 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1d8df958457324f8b4f19455d6dfd9fd859650b5ad024dd1fa064652bda3cd5 +size 1641243 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..c90b089512e096e9e5599306763f704e8f8a0b19 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera continues its arc, now showing the picnic table from a three-quarter rear view. The purple path is prominent on the right side of the frame, leading toward a small hill. Willow trees dominate the upper portion of the shot, with their branches creating a natural archway. The distant park benches are still visible, though partially hidden by intervening trees.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..069111aa45111ac8e4ff2a98bda0a21642833acd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c8c48508b5ff52c8983508e63fc1a3d24147b6548e7737dd7c6f124a1e5f769 +size 1861703 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..354109424046a094171688a288f924a1d0b7cdb2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera nears the completion of its circular pan, now facing the picnic table from a front-left angle. The purple path is visible on both sides of the frame, framing the scene symmetrically. The tree stump seats and stone table are fully illuminated by sunlight. The distant park benches appear smaller but still present, adding depth to the composition.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4dbc40925d75dd4b3165846feff2951b49262e09 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10cef499b3a278a224b0c7ddc6ae3d8a0429465437013d7e1ef04a66ff769753 +size 1815987 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..e9b238c7f56df9f4667c08939390a11be6b52d72 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera holds steady on a wide, balanced shot of the entire picnic area — stone table, stump seats, willow trees, and the purple-painted path curving gracefully on either side. The distant park benches are visible in the background, completing the tranquil park scene. The lighting remains bright and natural, with shadows gently shifting as the video concludes.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0889ba8a3f66b4f7ce328314dcc83c00028245e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fefa18e8c220e2166afa3d6225fdebbd85cc36f9bb49b0ecbcb17f5b884f5ea6 +size 1120466 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a4b047fad344796bfb556e5519a6cfc2073f161 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/console.log b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/console.log new file mode 100644 index 0000000000000000000000000000000000000000..d40f4b43ef9727a6512e0f6eb662a7bc5384c64d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/console.log @@ -0,0 +1,22 @@ +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/config.yaml +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.json +[02-20 10:58:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:12:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 11:12:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 11:12:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 11:12:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..40d5f1b2e78574b25d476520285949061851ffa5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/debug.log @@ -0,0 +1,353 @@ +[02-20 10:57:15|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 10:57:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:57:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:57:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 10:57:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 10:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 10:57:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 10:57:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 10:57:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 10:57:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 10:57:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 10:57:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 10:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 10:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 10:57:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 10:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 10:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 10:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 10:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 10:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 10:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 10:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 10:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 10:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 10:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 10:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 10:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 10:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 10:57:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 10:57:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 10:57:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 10:57:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 10:57:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 10:57:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 10:57:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 10:57:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 10:57:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:57:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:57:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 10:57:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 10:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:57:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 10:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 10:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 10:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 10:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 10:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 10:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:57:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:57:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 10:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 10:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.21 s +[02-20 10:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:58:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:58:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 10:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/config.yaml +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 10:58:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera continues moving forward, holding the dual-structure composition firmly in view. The curved metallic facade occupies the left third, while the glass tower dominates the right. Cloud reflections dance across both surfaces, and the green lawn remains consistently in the foreground. The video ends mid-motion, preserving the sense of ongoing exploration and architectural harmony.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.json +[02-20 10:58:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.mp4 +[02-20 10:58:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 10:58:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 10:58:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 10:58:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 11:12:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 11:12:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 11:12:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 11:12:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 11:12:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 11:12:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8cac94f7efd06d2eb52ce6789cd236d7d0a3d1f1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle, forward-moving shot across a neatly trimmed green lawn. A curved metallic facade with reflective silver panels appears on the left, featuring a circular window near its center. Clouds are reflected on the glossy surface, and a distant red building is faintly visible on the far left. The sky is partly cloudy, casting soft shadows on the grass.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb96ffe0e7b67768b1aaad80c98af026112cd73e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:141d6d1ab8a26b4ce2a791d718f13c14aeb57613e8703becdf68aa3da6f08446 +size 2068896 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..8d66ba1a05b3a88698e990320708993d27f54e58 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing forward, the camera maintains focus on the curved metallic facade while beginning to reveal the edge of a glass-paneled tower on the right. The green lawn remains prominent in the foreground. Reflections of clouds glide across the metallic panels as the camera advances smoothly, enhancing the sense of motion and depth.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e0b839c395d057002b91cadf9c7079767921ed9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c1a18d692c7e73431ce33cf466785a24a6eca48f7d4bf5b686b85234cab5d70 +size 1093035 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..1ef3b4cac829b8ab4161e6a4e87aa1c49718e921 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera progresses further, bringing the glass-paneled tower into clearer view. Its blue-tinted glass reflects the sky and surrounding structures. The circular window on the curved facade remains visible, and the reflective metallic panels continue to mirror the moving clouds. The distant red building stays in the background, subtly anchoring the scene’s spatial context.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f176b2581dcea201dfe4242fed75df06875d38a4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbd5d7a1881de692873def69553b7b75ecfde210c6c1f205597ff53de00f0f6c +size 873389 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..59039be192206ace6e02058bdb640b10767ce5b5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: As the camera moves ahead, the angle widens slightly to include more of the glass tower’s height. The curved metallic facade on the left curves inward, creating a sense of enclosure. The green lawn fills the lower third of the frame, and cloud reflections remain active on both building surfaces, reinforcing the dynamic lighting conditions.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4fb3687e312dcefb14474da5a6ecf2e64d917701 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c171241f1fa31ef0fb605d527268f9ca11b786ae95432a1d53020c0e59baf4e1 +size 791484 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c749f7fb1961008e7471c41f0e85fc47ebd7b793 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera continues its steady forward motion, now positioning the glass-paneled tower more centrally on the right side of the frame. The curved metallic facade recedes slightly but remains a strong visual element on the left. The circular window is still visible, and the reflective panels shimmer with cloud movements. The distant red building is less prominent but still present.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..36afd57e89abb5575737817e2508aee656301a91 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:667ba8c3522610bc83c78e7816628053ff84932db46c15afccd407e80dee2a53 +size 1088010 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..d1cab1dd7d5684b1e5d16b690c87db595ae747c6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The shot now emphasizes the architectural contrast between the curved metallic structure and the vertical glass tower. The camera subtly pans right, aligning both structures within a balanced composition. The green lawn stretches uninterrupted in the foreground, and cloud reflections animate the entire scene, creating a cohesive visual rhythm.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..27a6d7cb8704f1e27ba8441f271ab08f81d19746 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15cfdf1428621a86811a988d2ae5ef1f9a62bc3c883557f23d472581d9e536bf +size 1167824 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..d31e347d9f81d7d772e18e1d77aa9421f5897e91 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera maintains its forward trajectory, now showing the full verticality of the blue-tinted glass tower alongside the sweeping curve of the metallic facade. The reflective metallic panels continue to capture the sky’s changing patterns. The green lawn anchors the bottom of the frame, and the distant red building reappears slightly as the perspective shifts.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9fa70229fb8f260089bd80332d5a619709c9d701 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fa5fdc56963f4fa19072b9697b002d923ece414bd4689a39b01cddbd2de7b7d +size 1353784 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..1a997c0b0009926d210390e1c0690600864e2287 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera continues moving forward, holding the dual-structure composition firmly in view. The curved metallic facade occupies the left third, while the glass tower dominates the right. Cloud reflections dance across both surfaces, and the green lawn remains consistently in the foreground. The video ends mid-motion, preserving the sense of ongoing exploration and architectural harmony.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6a7e886b2416fd5f37cf818a8f81a417e572acf3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72101ecb116e37ad3b1b6f91e4fa68ce5df7b02f6453db41d1771d87e8796b14 +size 1372086 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5fcb8a565f646c157151de7e04b508747ff2f27 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/console.log b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/console.log new file mode 100644 index 0000000000000000000000000000000000000000..8fd2e46e7f61b68f4faf0b6b17dbe876ad6b96f4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/console.log @@ -0,0 +1,22 @@ +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:30:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:30:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/config.yaml +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.json +[02-20 17:30:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:44:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:44:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:44:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:44:10|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dd30b6f85edf3431449368517b32ca5e8274b77c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/debug.log @@ -0,0 +1,353 @@ +[02-20 17:30:03|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 17:30:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:30:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:30:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 17:30:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 17:30:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 17:30:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 17:30:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 17:30:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 17:30:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 17:30:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 17:30:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 17:30:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 17:30:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 17:30:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 17:30:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 17:30:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 17:30:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 17:30:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 17:30:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 17:30:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 17:30:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 17:30:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 17:30:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 17:30:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 17:30:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 17:30:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 17:30:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 17:30:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 17:30:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:30:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:30:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:30:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 17:30:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:30:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 17:30:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:30:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:30:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 17:30:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:30:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 17:30:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 17:30:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 17:30:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 17:30:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:30:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 17:30:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:30:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:30:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 17:30:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 17:30:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.39 s +[02-20 17:30:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:30:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:30:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 17:30:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/config.yaml +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 17:30:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Finally, the camera completes its orbit with a close-up of the lion’s face, emphasizing the intricate sculptural details — the furrows of its brow, the curve of its nose, and the depth of its eyes. The mane flows around its head, creating a halo effect. The tree, brick building, modern structure, and paved walkway remain visible in the periphery, maintaining environmental consistency. The grassy mound and mulch base are subtly present beneath the statue. The video ends with this intimate portrait, leaving a lasting impression of the lion’s noble and tranquil presence.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.json +[02-20 17:30:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.mp4 +[02-20 17:30:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 17:30:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 17:30:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 17:30:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 17:44:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 17:44:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:44:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:44:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:44:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 17:44:10|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0ee5f9b2e9f21517236fcaa0fb655f354e221fcf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a medium shot of a bronze lion statue lying on a mulch base, positioned on a grassy mound. The lion faces right, its tail curled upward. Behind it, a large tree with green leaves provides partial shade. To the left, a red brick building with white trim is visible. The surrounding lawn is vibrant green, edged with low shrubs. A paved walkway curves along the background. The camera begins a slow clockwise pan, revealing the lion’s muscular hindquarters and textured mane.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f5d12349a95a67650e69d0e026391225acb2f7fb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffc77f4e596d55c9e8c5bc211dd9591fd7ea36cd3d4484984c28c5ae7bbb1844 +size 2785463 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..78056f2aa0f895a2568c420eb6f191ba1f99f180 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its clockwise movement, the lion’s right flank comes into full view. The detailed mane flows over its shoulder, with individual strands clearly sculpted. The mulch base is circular and contrasts with the green lawn. The grassy mound slopes gently downward. In the distance, the modern multi-story building appears between trees. The paved walkway remains visible along the edge of the frame. The lighting is even, casting soft shadows across the statue and ground.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1c49ae91b11381191e878a408a3d80d9723a37b8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e19688c4bae6aaa2cd3706c73e52c649ea1001403b2e080f7636d04b45320781 +size 2238039 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..682f06dab5a3399a0adaf6815135c1e8f0c59b3a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera now captures the rear of the lion statue, emphasizing its powerful haunches and the curve of its tail. The tree behind the statue fills more of the upper frame, its leaves rustling slightly. The red brick building remains on the left, while the modern building is more centered in the background. The paved walkway curves out of view temporarily. The mulch base is fully visible from this angle, showing its neat edges against the grassy mound. The camera maintains a steady pace, preserving the serene mood.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..267582fd8ba12d88e72b3466d3d4850bf52ad8c5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68a9cb62508c85abe0abe001633f31cd8e5f77b0b2471ccaeaf37376f727c6e6 +size 1706397 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..89286c50694cdf31a91563a599d350e24f8eca60 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Moving to the lion’s left flank, the camera reveals the sculpted musculature of its torso and foreleg. The detailed mane drapes over its chest, catching ambient light. The grassy mound beneath the mulch base slopes gently into the surrounding lawn. The tree’s branches frame the top of the shot. The red brick building is now partially obscured by foliage, while the modern building remains visible in the distance. The paved walkway reappears along the right edge of the frame. The camera continues its smooth, deliberate rotation.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d733d31592deecff295a160ff2d337d4ae913da5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecf09151d5e57659e18bb6662bf61e0b79eb7eae71201cd48dcf3163d8f2024a +size 1005430 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..6af8c272ae32e28062348fab3d353fae56eeb631 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The camera now focuses on the lion’s front-left quarter, bringing its face partially into view. The detailed mane frames its head, with strands flowing downward. The lion’s eyes appear calm and watchful. The mulch base and grassy mound remain consistent in composition. The tree behind the statue still provides shade, while the red brick building and modern structure anchor the background. The paved walkway runs parallel to the lawn. The camera begins a slight zoom-in, preparing for a closer look at the lion’s features.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa9852539e8d83da53ca9b073c4804e83bdd1a14 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ac12c5747f2b0ec83b3297b6ffa768b280b0cf461224f30136e6da76073e9ba +size 869862 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..4d5a034781e4bfc624a141b7dbafde1f92dd7e93 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Continuing the clockwise pan, the camera centers on the lion’s face, now fully visible from the front-left angle. The detailed mane surrounds its head, with each strand meticulously rendered. The lion’s expression is serene, with softly modeled eyes and nose. The grassy mound and mulch base are clearly defined beneath the statue. The tree, brick building, and modern structure remain in their respective positions. The paved walkway is visible along the right side. The camera maintains its steady movement, enhancing the sense of reverence for the sculpture.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51cef2ab7e6f6be5c7d15c846cb4e2a5d13ce59d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7114fb46c89f5d45319cb8ee4c7469d5ae260778f404bc51d81ede3d3291624 +size 1000512 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..2ee4650513513ee08e9ab089fb56ac60ccf25d51 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"The camera now moves to a direct frontal view of the lion statue, capturing its full majesty. The detailed mane frames its face symmetrically, and its paws rest calmly on the mulch base. The grassy mound slopes gently outward, blending into the green lawn. The tree behind the statue provides a natural backdrop, while the red brick building and modern multi-story structure remain visible in the background. The paved walkway curves along the edge of the scene. The lighting remains soft and even, highlighting the bronze texture without harsh shadows.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a4fa2a5db26bbaa35c9069f5a8ad725e5d22b126 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8858b56738f93879035bc2017f623095de1edfdcb01420c8d4892026f9e76b5c +size 1614795 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..32d7fd7af72dd3606507510adceb082cf8711373 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Finally, the camera completes its orbit with a close-up of the lion’s face, emphasizing the intricate sculptural details — the furrows of its brow, the curve of its nose, and the depth of its eyes. The mane flows around its head, creating a halo effect. The tree, brick building, modern structure, and paved walkway remain visible in the periphery, maintaining environmental consistency. The grassy mound and mulch base are subtly present beneath the statue. The video ends with this intimate portrait, leaving a lasting impression of the lion’s noble and tranquil presence.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51c83147cf7c2d3cbbe6e3ee958e4336a2d3b01b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27a08d1c021e2e1e522e09d80ac8505a59b2bdff7ca01db73d0ae03660049c74 +size 1595457 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5e7a4b4443a7e13005fae71942800bdacc3d44d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/console.log b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/console.log new file mode 100644 index 0000000000000000000000000000000000000000..4a6e64f29e6f607b6f7b4a85090354634f5e10e7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/console.log @@ -0,0 +1,22 @@ +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/config.yaml +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.json +[02-20 19:25:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 19:38:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 19:38:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 19:38:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 19:38:31|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..22dc82bec87563fdf05c3a561dc37363ab425b98 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/debug.log @@ -0,0 +1,353 @@ +[02-20 19:24:22|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 19:24:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:24:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:24:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 19:24:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 19:24:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 19:24:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 19:24:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 19:24:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 19:24:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 19:24:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 19:24:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 19:24:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 19:24:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 19:24:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 19:24:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 19:24:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 19:24:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 19:24:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 19:24:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 19:24:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 19:24:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 19:24:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 19:24:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 19:24:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 19:24:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 19:24:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:24:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:24:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 19:24:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:24:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 19:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:24:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 19:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 19:24:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:24:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:24:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:24:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:25:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 19:25:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 19:25:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.70 s +[02-20 19:25:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:25:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:25:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 19:25:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/config.yaml +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 19:25:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: The camera completes its circular motion, returning to a frontal view similar to the opening shot but slightly adjusted. The red and white pole with lanterns is fully back in frame on the left. The asphalt road and surrounding trees create a balanced backdrop. The video ends with a gentle zoom-out, encompassing all mandatory landmarks in a serene, composed final shot.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.json +[02-20 19:25:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 19:25:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.mp4 +[02-20 19:25:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 19:25:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 19:25:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 19:25:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 19:38:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 19:38:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 19:38:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 19:38:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 19:38:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 19:38:31|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8621d9997a16820904dcd355a0d28deee21d7bc1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a static wide shot of two bronze ox sculptures lying side by side on a concrete pedestal with rust streaks. A red and white striped pole with hanging lanterns stands to the left. Behind the sculpture, an asphalt road runs horizontally, bordered by grass and scattered trees under an overcast sky.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa626bf2f4eb482dc8463d92a1b6f8abfea4023c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84c630abd21209d0650e1bbcd659e4cf57d2b5da52f0a88686c4342082d25286 +size 766883 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..39285754b172877d90f7e5d42aa2a9d521e8bbc9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera begins a slow clockwise pan, moving slightly closer to the sculpture. The front-facing oxen are clearly visible — heads raised, horns curving upward, tails extended. The red and white pole remains prominent on the left, while the asphalt road and trees in the background stay in view.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3a14a403a3b465d2d5634fb78d6ab01415922b00 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dc2862ffd946d70d5f39fa606f4d2831e16b9b242045cd8d7b3ee5cec7f0120 +size 814774 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..163e265fc26550cd6eaae1e99878dabd02ac20af --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: Continuing the clockwise movement, the camera shifts to show the left side of the sculpture. The red and white pole is now partially cut off at the edge of the frame. The texture of the bronze and the rust on the pedestal become more detailed. The asphalt road curves gently into the distance, flanked by leafy trees.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24a7ab2b3d1c0c5fec40877cafb13196521c7c52 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a00ac2a9a542aa630a8c09821c781a015bd0c7184685e3b7be4b083c9aadf447 +size 958705 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..95324c769a4ac0282ab96a86c6df8fe12b0d25b0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera reaches the rear-left angle of the sculpture. The red and white pole is no longer visible. The focus is now on the backs and tails of the oxen, with the asphalt road still visible behind. Grass and gravel surround the pedestal, and more trees fill the background, creating depth.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..edc9e852ab0042831d6b62f3d3b08f0147fc544f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fab094066f3b91e901be283093c4490f76ab6601b66bf34c27fb5bfd68649c48 +size 1426586 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..014cee130bb5bdb2f926be705d214e46f4611185 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: Midway through the orbit, the camera captures the rear-right side of the sculpture. The tails and hindquarters of the oxen are emphasized. The asphalt road remains in the background, though less dominant. Trees and shrubs dominate the frame, with occasional glimpses of distant structures through the foliage.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d391aec6403d73c67a794478ce1dd31853b425a4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a4cf45abc11e9136e525e5282d9841bb03f95970778e0a81e4abfe75acac105 +size 1569835 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..715a4509bec76885c1166be550e43a9589d8bc04 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera moves to the right flank of the sculpture, now showing the profile of the oxen from the side. The rust-stained concrete pedestal is fully visible from this angle. The asphalt road is still present in the background, but the surrounding trees and grassy areas take precedence in the composition.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7f21758bf3f132241bb9bad31df63ca2527c601 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7afe4efa729154024890bb5cff918688550badd0a6e5a44bf2726e0c3fd37e44 +size 1680707 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..2bc4a954b6b6eb1afc7c443fc82e69cdd4052625 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: Approaching the final quarter of the circle, the camera reveals the front-right angle of the sculpture. The red and white striped pole begins to reappear on the far left edge of the frame. The lighting remains soft and even, highlighting the contours of the bronze oxen and the texture of the pedestal.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7fb2264ddea408f7d4bd9c87efad693c546e1558 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3d9241917d2c0afd128089de1fb3d2575b4f75fd75a62beea735504118a4e45 +size 1388696 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..1453c6ba42cdea1571083e987db97839165b4ee9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: The camera completes its circular motion, returning to a frontal view similar to the opening shot but slightly adjusted. The red and white pole with lanterns is fully back in frame on the left. The asphalt road and surrounding trees create a balanced backdrop. The video ends with a gentle zoom-out, encompassing all mandatory landmarks in a serene, composed final shot.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9fd4d5317f41564287685592c222f065cc6f3d7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d72eb4fbbe9faf04325a21f0897e54677c83cb13e48def8960c5ce0e7e03fe1 +size 2041304 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf58ed5787bf5b42c0cde81d1cd34a4b600e4f67 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/console.log b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/console.log new file mode 100644 index 0000000000000000000000000000000000000000..252938e7b86750d5c9eba465a27a0bb31dd6f423 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/console.log @@ -0,0 +1,22 @@ +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:33:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:33:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/config.yaml +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.json +[02-20 11:33:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:47:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 11:47:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 11:47:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 11:47:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0231758cded2098c443031d95a6c9bffffd16d7f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/debug.log @@ -0,0 +1,353 @@ +[02-20 11:32:55|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 11:33:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:33:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:33:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 11:33:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 11:33:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 11:33:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 11:33:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 11:33:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 11:33:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 11:33:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 11:33:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 11:33:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 11:33:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 11:33:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 11:33:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 11:33:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 11:33:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 11:33:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 11:33:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 11:33:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 11:33:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 11:33:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 11:33:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 11:33:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 11:33:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 11:33:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 11:33:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 11:33:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 11:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 11:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 11:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 11:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 11:33:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 11:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 11:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 11:33:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:33:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:33:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 11:33:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:33:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 11:33:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:33:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:33:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 11:33:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.65 s +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.66 s +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.66 s +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:33:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:33:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 11:33:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 11:33:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.53 s +[02-20 11:33:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:33:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:33:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 11:33:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/config.yaml +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 11:33:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt="Part 8/8: Finally, the camera completes its full rotation, returning to a position nearly identical to the opening shot but subtly adjusted in height and angle. The white laundry basket with its floral pattern and the circular-patterned pillow are once again central. The beige leather sofa, black TV stand, and wall of framed pictures are all present, maintaining visual continuity. The lighting remains consistent, warm and inviting, and the TV screen still shows 'Upper East Side'. The smooth, continuous motion creates a seamless loop that encapsulates the entire living room environment." negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.json +[02-20 11:33:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.mp4 +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 11:33:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 11:33:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 11:33:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 11:47:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 11:47:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 11:47:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 11:47:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 11:47:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 11:47:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e7e20d486035e9da3bcba1bf018408cf85b40b6e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle shot focused on a white laundry basket adorned with a colorful floral pattern, resting on a white metal drying rack. The basket contains multiple pillows, including one with large, overlapping circular designs in earthy tones. To the right, a beige leather sofa is partially visible, with a light blanket draped over its arm. In the background, a black TV stand supports a television screen displaying the text 'Upper East Side' over a city image. The wall behind the TV is lined with framed pictures of various sizes.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22d0869df57682b6011666f9ef8964b75e34a29b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:495fe93283a6e562d64fb7b421ac0d8b033ebdcac5b0a40b9d3693e6a10d0374 +size 783587 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..d6a19abd95d0ac4545455d53e8f4683ebcb6ca16 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera begins a slow clockwise rotation around the laundry basket. The beige leather sofa comes more fully into view, revealing its curved shape and plush cushions. The blanket remains casually placed. The TV screen continues to show 'Upper East Side', and the framed pictures on the wall become more detailed as the camera moves. Natural light filters in from a window with sheer curtains, casting soft shadows across the carpeted floor.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c569ee1e44038e4182e49f5e39d6289deb9f5afd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b7a9846d63226498dea482397b5feeedce119f1eb04c061edb3440ff91066b8 +size 1235153 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e09f07ac29b9d5d56b0774c5aeed23db1943f67c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: Continuing the rotation, the camera angle shifts to highlight the side of the circular-patterned pillow, emphasizing its vibrant colors and geometric design. The white laundry basket’s floral pattern is now seen from a new perspective. The black TV stand and the displayed text remain visible, though slightly angled. The framed pictures on the wall appear in a diagonal line due to the camera’s motion, adding depth to the scene.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f549ee9cca07aacf1160c19b3a421f6cf7a1cd1b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:701cb528944c9a703db46ddbeccc1e8d66000185bb87b87ae9aedb1df6664465 +size 1023702 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..56369ca8ed70e2619845abfc136d2321a3bd05e8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera tilts slightly upward as it rotates, bringing the top of the drying rack and the piled pillows into sharper focus. The beige sofa is now seen from behind, with its backrest and armrests framing part of the shot. The TV screen remains illuminated with 'Upper East Side', and the wall of pictures is now viewed from a lower vantage point, making the upper frames more prominent. The ambient lighting from the floor lamp enhances the texture of the sofa and the rug.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..763a29f34684e8a62e2b727667c77c09ad64769d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98fb1f861bcf2725441eb30349477b49017a5d297a1940ccafa64c4d045c31f9 +size 823096 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..dadf41b503fca482c8966198333c98fe332b9914 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: Midway through the rotation, the camera offers a frontal view of the laundry basket, now centered again but from a slightly elevated angle. The circular-patterned pillow dominates the foreground, its bold design contrasting with the delicate floral pattern of the basket. The beige sofa is visible to the left, and the black TV stand anchors the background. The framed pictures form a vertical array along the wall, each frame distinct yet harmonious in the overall decor.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1cd4edab9af88a065c42d7d73dbf2ff2ca289023 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fd9abca8def1ae1e0e9e383c7e19a10c3e15ef4225e5d4083d6f56a38e2a21d +size 794646 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..260ef2a83227220cb3a21d349e65d265e9247172 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: As the rotation progresses, the camera captures the rear of the drying rack and the underside of the laundry basket, revealing its plastic grid structure. The circular-patterned pillow appears from above, its pattern appearing compressed. The TV screen still displays 'Upper East Side', and the beige sofa is now seen from the far side, with its cushions and blanket softly lit. The framed pictures are viewed from a reverse angle, showcasing their arrangement from a new perspective.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8770217558ec981e674719fa372ace1a5fbe08f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18d8f866655cd4a650991b1d113fa8ec95468f3b323edcebb5e387b693df01ab +size 1194379 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..1d929903fdc3894ddcf0197b25030fb9b33eb72b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera continues its clockwise motion, nearing completion of the 360-degree circle. The white laundry basket reappears from the front, now slightly lower in frame. The circular-patterned pillow remains a focal point, its colors vivid against the neutral tones of the room. The black TV stand and the 'Upper East Side' display are clearly visible, and the beige sofa returns to prominence on the right. The framed pictures on the wall are once again aligned horizontally as the camera approaches its starting orientation.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22cb5f62b40256456587dd84621c862ffca66385 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17fbe8029af62df8e02bf8503951568dde75be3a308b2e4c96af60c4dd3e1199 +size 1220033 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..ab32e495d7208846e4d8d927c2a6ed240aae3ab6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its full rotation, returning to a position nearly identical to the opening shot but subtly adjusted in height and angle. The white laundry basket with its floral pattern and the circular-patterned pillow are once again central. The beige leather sofa, black TV stand, and wall of framed pictures are all present, maintaining visual continuity. The lighting remains consistent, warm and inviting, and the TV screen still shows 'Upper East Side'. The smooth, continuous motion creates a seamless loop that encapsulates the entire living room environment.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..16d45c48731166781792a9c29819de5e16aa6f2b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6357747dd15549a232e60470924288bc9b5e175f8eaebb15e986c29f0c8493 +size 949202 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..851c779c120f37bd921bf3861096ee2babeaccf1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/console.log b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/console.log new file mode 100644 index 0000000000000000000000000000000000000000..387d83a98740b9242f6a3fd58aab939f288dc487 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/console.log @@ -0,0 +1,22 @@ +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:00:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:00:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/config.yaml +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.json +[02-20 13:01:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:15:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:15:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9f1b35a831df816638dc28ed5bcd5cc53742084b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/debug.log @@ -0,0 +1,353 @@ +[02-20 13:00:26|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 13:00:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:00:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 13:00:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 13:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 13:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 13:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 13:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 13:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 13:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 13:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 13:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 13:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 13:00:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 13:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 13:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 13:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 13:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 13:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 13:00:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 13:00:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 13:00:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 13:00:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 13:00:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 13:00:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 13:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 13:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 13:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 13:00:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 13:00:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 13:00:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 13:00:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 13:00:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 13:00:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 13:00:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:00:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:00:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 13:00:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:00:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:00:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 13:00:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:00:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 13:00:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:00:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:00:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 13:00:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:00:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:01:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 13:01:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 13:01:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.83 s +[02-20 13:01:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:01:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:01:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/config.yaml +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 13:01:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: The video concludes with a final wide shot as the camera stops at the end of the walkway. The entire pavilion is centered in the frame, surrounded by natural elements and juxtaposed against the distant city skyline. The scene captures the quiet beauty of the structure nestled between nature and urban life, ending on a note of serene balance.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.json +[02-20 13:01:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.mp4 +[02-20 13:01:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 13:01:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 13:01:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 13:01:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 13:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 13:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:15:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:15:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 13:15:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..4a3fc174627bdff0e99441615d616fda9237ba18 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a wide shot of a wooden walkway extending toward a hexagonal pavilion, flanked by tall reeds and calm water. In the background, a distant city skyline with high-rise buildings is visible under a clear blue sky. The camera begins a slow forward movement along the walkway, passing beneath leafy tree branches on the left.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8236069f564f8f7472f7a674dc6c999a6e41505 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02cb0375ca81bda65c1168fa22a6eb0fd001ae727efc2685cb99a498ff83b8b9 +size 1692495 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..13f806192086444e89e3b8e447dbb4593043b05c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing forward, the camera approaches the entrance of the pavilion. The decorative eaves of the hexagonal roof become more prominent, and the wooden railings on both sides of the walkway are clearly defined. Reeds sway gently beside the water, and the city skyline remains visible through gaps in the foliage.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c015340fecd99bc9ce1ea32c3f6af2672f1945f3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76743a21d3915b8b6491553db6b8c15bd238f35f492d2950c1a381ffc8da9a94 +size 1563973 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..1b40e05ef64d91d145833d6fe8e0decbbbe849f7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera enters the pavilion, revealing its interior. The roof structure — with layered beams and ornamental trim — is showcased as the camera tilts upward. Wooden support pillars frame the view, and metal benches are visible along the inner perimeter. The wooden floor planks appear smooth and well-maintained.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d9cae0e851b64d4ab649d1b7652729d2b6e39422 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e2d425dabef7111c13ac96c6e37eb097366bda652a591460fce39ec535134af +size 1853578 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..d6824bf29d5ceee354f56918c2c3d8281a5ef4ca --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: Inside the pavilion, the camera pans slowly to the right, capturing the arrangement of benches and the open views through the railings. The distant city skyline is framed by the wooden railings and surrounding greenery. Sunlight filters through the roof structure, casting soft shadows on the floor.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64c884220f48cc22f87ea851ab7a10bbe135a675 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6764cc7736ddc33e04ad5b9b6330cd6fd37b52b603b8ad97bcd64e210a2ffa28 +size 1373686 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..da702bea0e41636dcf74666f82f0eb8dbdeea1e3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera continues its circular pan, now facing the opposite side of the pavilion. The view reveals more of the lake and the dense reeds bordering the water. The wooden railings remain consistent in design, and the hexagonal shape of the pavilion is emphasized by the symmetrical layout of the benches and pillars.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d30871d6f5f09d4f50dca8c4442701de47491b1c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be3d3f522517d5ca1bc9738854ee22cc25a385837bad59b3a70f69e199be6183 +size 1232660 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..71c5f85dafcc1b76ef91c1cd99655c26104a64fd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera tilts downward to focus on the wooden floor, then begins to move backward toward the entrance. The benches and railings recede into view as the camera retreats, maintaining the perspective of the pavilion’s depth and spatial layout. The reeds and water on either side come back into frame.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7800aaf2ac4c4cb644c6c42cd2f35f25968f292a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66d0d119b29f2eec88404af884b9bd2261db5f7f8fe9adc66343d726a20d492a +size 1352629 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..0abce57ca842a3dc8b9bdf52e048a2682c7d25e6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: Exiting the pavilion, the camera continues to pull back along the wooden walkway. The hexagonal roof and decorative eaves are now seen from behind, with the city skyline once again becoming the dominant background element. The wooden railings guide the eye back toward the starting point of the walkway.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da5450fd209c8645727c8c9b5606fa88048a8727 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f43fd94ae65fca9170abf4d96b6fd36156d8afba386e6eb874b5930c9ad582d +size 1803115 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..8d81618cece7f87ced3f4646a9ae4b7e6bdc1ab1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: The video concludes with a final wide shot as the camera stops at the end of the walkway. The entire pavilion is centered in the frame, surrounded by natural elements and juxtaposed against the distant city skyline. The scene captures the quiet beauty of the structure nestled between nature and urban life, ending on a note of serene balance.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5921845277d55907fc8fea327bf7d246ba0a4349 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16f44710dcc66813154925690df38d8d40f1e048233fa5f8b878cdbe0a557137 +size 974668 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..963a75eabe1de88a6a9be060cd420041db665766 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/console.log b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/console.log new file mode 100644 index 0000000000000000000000000000000000000000..e19ebb1675ed24ee9ac9f1e95f38824cbd3d8ecc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/console.log @@ -0,0 +1,22 @@ +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:08:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:08:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/config.yaml +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.json +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 19:22:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 19:22:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 19:22:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 19:22:41|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..993e5e6517dd581591ec08c27f571b7449817a21 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/debug.log @@ -0,0 +1,353 @@ +[02-20 19:08:34|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 19:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 19:08:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 19:08:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 19:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 19:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 19:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 19:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 19:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 19:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 19:08:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 19:08:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 19:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 19:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 19:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 19:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 19:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 19:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 19:08:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 19:08:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 19:08:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 19:08:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 19:08:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 19:08:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 19:08:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:08:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:08:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 19:08:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:08:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:08:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 19:08:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:08:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 19:08:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:08:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:08:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 19:08:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:08:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:08:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:08:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 19:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 19:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.61 s +[02-20 19:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:09:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:09:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/config.yaml +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 19:09:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera pulls back slowly, encompassing the entire showroom in a wide-angle shot. The full expanse of the space is revealed — from the entrance area with the round mirror and console table to the farthest furniture vignettes. The warm lighting, natural materials, and balanced color palette culminate in a cohesive, inviting retail environment designed for immersive browsing.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.json +[02-20 19:09:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 19:09:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.mp4 +[02-20 19:09:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 19:09:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 19:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 19:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 19:22:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 19:22:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 19:22:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 19:22:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 19:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 19:22:41|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c33b49e56c830f6ebfbe7953201768a24d36aff2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a smooth tracking shot moving left to right along a white shiplap wall. A large round mirror mounted on the wall reflects ceiling lights and nearby plants, creating soft shimmering reflections. Below the mirror sits a wooden console table with a tall vase containing dried branches. The camera begins to move forward slowly, revealing polished wood flooring and ambient lighting from recessed ceiling fixtures.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1b2e4eaf6b6df0ab5e365a316e776d9484b05b65 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fd32c90d202c7da44a07effcfbb4cdaaef6e94331d5e075677ca7830a38aa6a +size 1005780 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..1e2c17d8ef34bf1dbe44d460d26b766143494332 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing forward, the camera passes a woven armchair draped with a mustard-yellow throw blanket. To the right, a large potted plant with broad green leaves stands beside a low-profile dark gray sofa. The camera pans slightly right, revealing more of the open-plan showroom layout with multiple furniture vignettes arranged under warm overhead lighting.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..198bc2eba5afae4b9aad18a660d6bd57af7afd70 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d53aa33569938260256e78311c47bb2d5032b0ceaf483d6c0c26dec5bac0e784 +size 1644128 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..80c3d918851be37d792b710bb5ec4720862afaea --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera glides past a freestanding white display panel featuring a prominent sign reading '20+ fresh tonal shades'. Behind the sign, a partial view of a seating area with neutral-toned upholstery and accent pillows is visible. The camera maintains steady forward motion, transitioning into a wider aisle lined with curated furniture displays.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6a5696de3f7d01d11c5b4472049ddb5ba4fa4c80 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f322f83ba5e90c25deb470abe6f8d31b7d416b5a9f20fde1f943cae67db33014 +size 1426266 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..2c2c330c7266985b70e04e7c6a0ad94dbbfd6728 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera moves deeper into the showroom, revealing an expansive aisle flanked by sofas, side tables, and decorative lamps. Recessed spotlights illuminate each vignette. The camera pans slightly left, bringing into frame a tall rectangular mirror mounted on a white pillar. The mirror reflects an adjacent aisle lined with metal shelving units displaying folded textiles and decorative pillows.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..546bb96e9f549bbfc5be8eb50abe12f438ce0324 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b58c9d32e00e544d6cb2ae3fcd989e65bf1f6bedc5245bcfbfe5563f05b495a +size 1341905 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..574e2b422909982703b2a4074e1c429d5e9dc05c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: Focusing on the tall rectangular mirror, the camera holds briefly to emphasize the reflections of ceiling lights, distant furniture displays, and the organized shelving behind. The reflection creates a layered visual effect, adding depth to the scene. The camera then begins to turn left, preparing to circle back toward the entrance area.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92bceec09ac6056ba000cfdf36eee17688dc6af9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e2ac472621bd6846bc5b41b06eca4b0969e6583f6aee12d853d9af2d62e1520 +size 2033963 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..d6d92244b18196282c17537d0d0ec7545bb14110 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera turns left and glides backward slightly, offering a new perspective on the round mirror previously seen at the start. Now viewed from a different angle, the mirror again reflects ceiling lights and surrounding furniture, reinforcing the showroom’s symmetrical design. The wooden console table with the vase remains visible beneath the mirror.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..52315a99f28f508761e79eb55ab0f096b030b18b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50b8f2dded3f9293be6e6491205bb2a1c0eb6b1e79b663f626495fe7011edfa6 +size 1717392 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..a7f8414768fc2661d730fc44869c3f4f4957c189 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera sweeps horizontally across the showroom floor, capturing the warm wood flooring, high ceilings with exposed beams, and strategically placed indoor trees. Mirrors throughout the space reflect light and furniture, enhancing the sense of openness. The camera movement is slow and deliberate, allowing viewers to absorb the curated aesthetic and spatial harmony.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0bdb9c1e96715bf4086e36042c36faa90eb7cb4c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac4768b5e99300896f16d21e310cb7671c5813246c92774e6eaadc8ae3d18db +size 1686478 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..e9cf7bd341590386886b38604720ce04eb1ac866 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera pulls back slowly, encompassing the entire showroom in a wide-angle shot. The full expanse of the space is revealed — from the entrance area with the round mirror and console table to the farthest furniture vignettes. The warm lighting, natural materials, and balanced color palette culminate in a cohesive, inviting retail environment designed for immersive browsing.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a8f2c788946b499a92ade4fdeb67111f95c1c08 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:024b10aa38be53086185ac8767691a7e8a16d49953a14c5bef769eb5ffddada7 +size 1391959 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d04f607945cdf370b58791cfcb4c6fa8a6a09a32 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/console.log b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/console.log new file mode 100644 index 0000000000000000000000000000000000000000..87cb16b3ce5a414ca352350d2922afb859a3b08e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/console.log @@ -0,0 +1,22 @@ +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 15:36:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 15:36:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 15:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/config.yaml +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.json +[02-20 15:36:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:49:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:49:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6969304b7308374d531fc0720e1da45f603ef36d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/debug.log @@ -0,0 +1,353 @@ +[02-20 15:35:47|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 15:35:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 15:35:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 15:35:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 15:35:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 15:35:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 15:35:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 15:35:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 15:35:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 15:35:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 15:35:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 15:35:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 15:35:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 15:35:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 15:35:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 15:35:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 15:35:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 15:35:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 15:35:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 15:35:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 15:35:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 15:35:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 15:35:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 15:35:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 15:35:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 15:35:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 15:35:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 15:35:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 15:35:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 15:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 15:35:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 15:36:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:36:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 15:36:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:36:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 15:36:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 15:36:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 15:36:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 15:36:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 15:36:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 15:36:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 15:36:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 15:36:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 15:36:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 15:36:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 15:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 15:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 15:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:36:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 15:36:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 15:36:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.12 s +[02-20 15:36:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:36:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:36:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 15:36:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/config.yaml +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 15:36:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera tilts downward, focusing on the base of the ironing board and the surrounding beige carpet. The patterned pillow remains in view at the top of the frame. The video ends with this grounded, intimate shot, reinforcing the room’s calm, cohesive aesthetic and leaving the viewer with a sense of spatial completeness.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.json +[02-20 15:36:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.mp4 +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 15:36:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 15:36:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 15:36:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:49:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:49:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:49:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 15:49:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..272bda50e73305544cdd8aa0c04c7e0e398717bc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle, wide shot of a living room centered on a white ironing board holding a patterned pillow (purple, orange, brown circles). To the left, a beige sectional sofa with coordinating pillows and a floral blanket is visible. Behind it, a wall displays framed pictures. Natural light filters through sheer curtains near a large TV showing a cityscape. A blue exercise ball sits near the TV stand.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09cf0395eb66b54d4775606e3b56d0ab3214323a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77368392dd1c5b47912df7165d338b10c2c3da2995790c250c14c67955de9140 +size 951425 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..cc5f29de701c1a0ffd1317a8579e54291420df9e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera begins rotating clockwise around the ironing board, revealing more of the sectional sofa’s L-shape. The TV remains in view, displaying skyscrapers. The camera angle stays low, emphasizing the plush beige carpet and the ironing board’s metal legs. A floor lamp glows softly in the background, casting warm light on the sofa.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..91e6161a8546f7dc2c34abb28701d2a25dea3569 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cae9faddcddd7a683d178f6f18aa1d568fa7669098f35dcc3c41671c41ccaf83 +size 1051199 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..cf3779b775f9f8030e9f257029a837d161c129cb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: Continuing the rotation, the camera passes behind the ironing board, offering a clearer view of the framed picture wall above the sofa. The blue exercise ball is now fully visible beside a black side table. The TV screen remains bright with the cityscape. The camera maintains steady motion, keeping the ironing board centered as the focal point.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1046f6ee2da38b281c71a2337e7a7a39582a8d01 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d37648c50ca5ab77bb94e020fcc7424ae0b7feadecf8a5bbb90807014653006d +size 1016450 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..c04ea383ceaff076d636b8209f21f7fa014850fb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera moves past the sofa’s corner, revealing a beige armchair with a decorative pillow and a small wooden end table. A red backpack rests on the floor nearby. The ironing board’s legs and the patterned pillow remain prominent. The camera begins to tilt slightly upward, drawing attention to the pillow’s design.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37bc6fdedc887f4835b235122d9d4c7bab36b41d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9584f0e2646db9a450cbcc78ad569bb473ee0b7d1cc0b228dd6b0d3832faa560 +size 1300425 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..794e1cbd6afa13b22a8b154b9f3139fa2ee8315d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera tilts up further, focusing on the pillow’s texture and pattern, then pans down along the crisscross metal legs of the ironing board. The plush carpet fills the lower frame. The TV and cityscape are still visible in the background, along with the blue exercise ball and framed wall art. Lighting remains consistent and warm.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37351606e28f0bcfc90c77cf5252fd7adc2f2cfb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8847505918d9c050fd3e15c238758f1a61e2dfc87ad5cb8cb59504ade3fafd98 +size 1736984 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..986fcec37c2ec1cf45ce49c82f07b52e781b3c15 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera sweeps back up and continues its clockwise rotation, now capturing the opposite side of the room. The beige armchair and end table reappear, along with the red backpack. The ironing board remains central. The camera movement is smooth, maintaining spatial continuity and emphasizing the room’s symmetrical layout.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7cb5e2baa8e2cf0faac1d2b0a625c47ebe10500d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4f1f320f7d4bd07e30beaed6ea0425d93930b0daf5e1433a939d2b41789a0ee +size 1882021 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..1013fc5f5e721cb04ef76b32a9163146b564cd2e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera completes nearly a full 360-degree orbit, returning to the initial viewpoint but now zooming in slightly on the ironing board and pillow. Shadows from the room’s lighting accentuate the pillow’s contours. The TV cityscape, blue exercise ball, and framed wall pictures are all visible in the periphery, anchoring the scene.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d0c370d52131127a41c59fde87e8f6125bd7925a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d1fe01e2740bbcb5f87c6c0aaeb07eef7b3934bf423f5403786ee50de468c53 +size 1624226 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..7b6fa096f90666afcc2e42e97adb61fae0847892 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera tilts downward, focusing on the base of the ironing board and the surrounding beige carpet. The patterned pillow remains in view at the top of the frame. The video ends with this grounded, intimate shot, reinforcing the room’s calm, cohesive aesthetic and leaving the viewer with a sense of spatial completeness.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..db64a2fbc5e24950b1ede3a963796544bc1108a5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39ccf028c0e6a1de6dce4151c8f72f85e43c3060c97dbe0642076c24623b942b +size 2112837 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f110e6ea2afe88e0f8a133ea8a971d942a146904 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/console.log b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/console.log new file mode 100644 index 0000000000000000000000000000000000000000..d97f8921f2bda6c1cb12afa9a60e81a3125c4700 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/console.log @@ -0,0 +1,22 @@ +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/config.yaml +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.json +[02-20 03:47:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:00:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:00:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3b9d36e1c6b3e222263c5a83f6a09abf1003e40f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/debug.log @@ -0,0 +1,353 @@ +[02-20 03:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 03:46:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:46:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:46:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 03:46:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 03:46:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 03:46:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 03:46:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 03:46:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 03:46:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 03:46:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 03:46:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 03:46:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 03:46:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 03:46:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 03:46:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 03:46:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 03:46:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 03:46:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 03:46:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 03:46:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 03:46:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 03:46:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 03:46:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 03:46:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 03:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 03:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 03:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 03:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 03:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 03:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 03:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 03:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 03:46:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 03:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 03:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 03:46:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 03:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:46:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 03:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:46:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 03:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.05 s +[02-20 03:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 03:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 03:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 03:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:46:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:46:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:47:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 03:47:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 03:47:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 25.55 s +[02-20 03:47:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:47:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:47:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/config.yaml +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 03:47:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera executes a slow, wide pan that frames the black bull statue perfectly between the Eiteljorg Museum on the left and the Indianapolis Colts skyscraper on the right. The overcast sky provides a neutral backdrop, and the entire scene conveys a sense of civic harmony and urban design integration, concluding with a static, balanced composition.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.json +[02-20 03:47:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.mp4 +[02-20 03:47:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 03:47:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 03:47:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 03:47:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 04:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 04:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:00:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 04:00:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e93350440892d12022a88980088c26c8c3bd58b6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle tracking shot moving forward along a grassy slope bordered by thick green shrubbery. To the left, a tall red sandstone wall stands, while ahead, the beige geometric facade of the Eiteljorg Museum begins to emerge under a cloudy sky. A large black bull statue on a red pedestal is partially visible behind foliage.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e2c247084a78007f2cd8bdcc88e3083d6ab5f4f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5484fc31355b2191b67bf3ade117b3ef52392c0ffd9369aa82962422e1bb347 +size 1743788 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..73bc852eaae039eec1ed6af48843cf7e40883f84 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera continues forward and pans right, bringing the full form of the black bull statue into view. Its muscular body and upward-pointing horns are emphasized as the camera circles slightly. The Eiteljorg Museum’s entrance and signage become clearer, with windows and overhangs visible beneath the overcast sky.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb91ca387e8f54694ca403ca20a31b5730cf40b3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f725779011cc1dcc3a01ae89105b6eacc54c9f4737d6557be921bb4d39739499 +size 1500344 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e65fa3543fc758b9ba164641c2d8b20dc360cb64 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera completes a partial circle around the bull statue, now showing its left flank and rear. The museum’s modern architecture fills the background, with clean lines and recessed entryways. Shrubs and small yellow flowers at the statue’s base add color contrast to the earth-toned materials.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bdee775c283c1a96e50dea15ccfb40bda6920306 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98af73d3126e6aa06e9b274d92a5076569f51a2f6e47daae85e5d24f58a3c943 +size 1376127 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a07aa39bdf08aec02974448a25d89e6834a4a78b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera continues its arc, now revealing the open plaza behind the statue. Paved walkways, low stone walls, and distant buildings come into view. The sky remains overcast, casting diffuse light that softens shadows and highlights the textures of stone and metal surfaces.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3d927789ccfb114beb309015c4f10963099efbd3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9827cb9b0fbb414540acbb30bfb571fc0cf880b93c25f0cfa53275d92ef76315 +size 2206022 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..d94783c25c4fddbae59f37405b18ce0ea3d935bb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: As the camera pans further right, the blue Indianapolis Colts skyscraper suddenly enters the frame, its vertical white text 'INDIANAPOLIS COLTS' standing out sharply. The camera tilts upward briefly to capture the tower’s height, then returns to eye level, framing the bull statue with the skyscraper in the background.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e75420e3cbe90b2f1b7f33229997a83c08955d56 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71c465aa1f08458201044ae981507ef25c329dc83294b5c037ca053c4b7faa2 +size 1996823 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..6e144b1f950304bc5d37c0bb078761b5e6776485 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera circles back toward the museum, now showing the bull statue from a rear-three-quarter angle. The Colts tower remains visible over the statue’s shoulder, creating a visual juxtaposition between the artistic sculpture and corporate architecture. Landscaping elements like trimmed hedges and lawn areas are clearly defined.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f33ceb1d8ace6389c0bee866b00b899686d5e35c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf9335b523c8d4d6ee078dd9d4070b083e6969c2c784e263b8d732064405370f +size 1747025 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..17c3ccb6a7bc7d7459dec753a0182894efc40375 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera lowers slightly and moves closer to the statue’s base, capturing details of the red sandstone pedestal and surrounding ground cover. It then pulls back slowly, re-centering the statue within the wider context of the plaza, museum entrance, and adjacent buildings.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..26f896785c3e3bf1026ad3e5b72143502a60a1fa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6e94653ce9b83dbb1562d22f588d3624dde76374615fb45dd22925c70121aa0 +size 2248958 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..49b963691a7fe676f1e626149d9aa35c19bfe834 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera executes a slow, wide pan that frames the black bull statue perfectly between the Eiteljorg Museum on the left and the Indianapolis Colts skyscraper on the right. The overcast sky provides a neutral backdrop, and the entire scene conveys a sense of civic harmony and urban design integration, concluding with a static, balanced composition.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bcf83d28fcc29ca6867c853c40d7c1317d4b3ca5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60c32e34902e6a663e3995cec8fcbd2c09d07e60165f24ce6818af457b7c7946 +size 2171183 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17471f0e65a1f62988fa0d42264497a3342b957f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/console.log b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b9d3632c94276b292f98757a43181703d5332c40 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/console.log @@ -0,0 +1,22 @@ +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:35:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:35:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/config.yaml +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.json +[02-20 12:36:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:49:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:49:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:49:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:49:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6f3b172019ea23665a946226ca21038dfbf188d5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/debug.log @@ -0,0 +1,353 @@ +[02-20 12:35:29|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 12:35:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:35:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:35:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 12:35:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 12:35:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 12:35:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 12:35:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 12:35:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 12:35:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 12:35:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 12:35:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 12:35:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 12:35:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 12:35:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 12:35:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 12:35:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 12:35:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 12:35:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 12:35:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 12:35:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 12:35:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 12:35:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 12:35:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 12:35:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 12:35:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 12:35:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 12:35:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 12:35:41|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 12:35:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:35:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:35:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:35:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 12:35:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:35:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 12:35:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:35:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:35:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 12:35:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:35:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 12:35:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 12:35:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 12:35:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 12:35:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:35:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:35:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 12:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 12:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.63 s +[02-20 12:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:36:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:36:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 12:36:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/config.yaml +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 12:36:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its circular motion, returning to a position similar to the start but slightly elevated, showing the front-left corner of the green picnic table once more. The brick wall, paved ground with square tiles, and distant building entrance are all visible, creating a cohesive and complete environmental shot. The camera slows to a gentle stop, concluding the sequence.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.json +[02-20 12:36:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:36:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.mp4 +[02-20 12:36:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 12:36:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 12:36:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 12:36:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 12:49:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 12:49:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:49:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:49:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:49:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 12:49:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ffc8ed4e0a6b993678559d1a5fc05e7d5639293b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle shot focused on the front-left corner of a green octagonal picnic table with black metal legs, positioned on a paved ground made of square beige tiles. A red brick wall is visible to the right, and in the far background, a distant building entrance with white-framed doors can be faintly seen. The camera begins a slow clockwise rotation around the table.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a884bb4f29aa211d77efb423de902d00751802a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d16a8e20d9aabfbfc24edd23c4353eb7d2b465017deeadf4ebc45df300e16f55 +size 874327 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9c8db059865df4d499bda332a64bd532c8da227e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the clockwise pan, the camera now reveals the full left side of the picnic table, highlighting the attached curved bench and the slatted tabletop. The brick wall remains in view on the right, while the paved ground extends forward with uniform square tiles. The distant building entrance becomes slightly more defined as the camera angle shifts.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ecf1bd67c7013b607d9ef9e876d007ff4ff404d2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b064f6879b395eb38b49983e37f98afd921521df44a9b5dff5822f07cb0fc70 +size 1062425 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d4470b9a40c8ab4d8db85c5d2bfb0c25b33f779d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera now captures the rear-left quarter of the picnic table, showing the symmetry of the bench seating and the black support structure underneath. The brick wall still occupies the right edge of the frame, and the tiled pavement continues to stretch outward. The distant building entrance is more clearly visible, with steps leading up to the white-framed doors.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..528bc4750c97c34058824680636a248a7ec46d15 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f9e5a84a22bdd3652575c75c1d0a26d4b541c9aff90fbeea3e58b4fa6b8cb35 +size 777733 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..d8f8d244d9ad9ddff89b5a51a77c8e1808abc8e8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: As the camera moves to the rear of the table, the full backside of the green picnic table comes into view, emphasizing the octagonal shape and the integrated bench design. The brick wall is now partially out of frame, replaced by more of the paved ground. The distant building entrance remains visible in the background, centered in the upper part of the frame.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7044275be93830107e02b50efa1f26ba2bd97db2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79c7391b2bb5114ff5744a9f0561322b0559d3f3769dbaa605f5a25cc3f5ce29 +size 1039841 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..eb6305e55ec9b1a5302b143c618e9f2d7b3ba6d3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera continues its clockwise arc to the rear-right side of the table, showcasing the right bench and the black leg supports. The brick wall reappears on the right edge, and the tiled pavement stretches diagonally toward the foreground. The distant building entrance is still visible, though slightly obscured by the table’s structure.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d44134e1d1dd48420895726aa2986935b4a140a8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aed5fc1f8815edcc116d6fc1bc454d3bee1da15b82e7c6b5156179c06a3b0ae5 +size 994640 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..7ca8c88bcfc78cb372a1bd96bf2e01018b9a4666 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: Now capturing the right side of the picnic table, the camera highlights the curvature of the attached bench and the slatted top. The brick wall runs vertically along the right edge, while the paved ground fills the lower portion of the frame. The distant building entrance remains in view, framed between the table and the wall.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e1256cded4c575143dc146162b6db02cfc7a7073 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c4fa660ec1bcd3f646e964d3091c26b37b74ec3e7f7c61854add18e141c0006 +size 1035814 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..84c96ad8f11d3d8f8518da43449f7758644fb515 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera moves to the front-right angle of the picnic table, offering a view that combines the tabletop, bench, and black support legs. The brick wall is prominent on the right, and the tiled pavement extends forward. The distant building entrance is still visible in the background, now aligned with the center of the table.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70023220f74f9d0b2c83fbb25afd1554726e177a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5690380a1611a6c52d4f94ddce2e8dfe7277a411a5aeafbb146f3e7ea9c58690 +size 1169764 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..5cd471f3e5c54731a3ef959309c559c3713c5fd8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its circular motion, returning to a position similar to the start but slightly elevated, showing the front-left corner of the green picnic table once more. The brick wall, paved ground with square tiles, and distant building entrance are all visible, creating a cohesive and complete environmental shot. The camera slows to a gentle stop, concluding the sequence.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a3eb5238cc1dba34b09d274f7eef3aaf167a62b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:146bfb68b440aeaa3854540f8ee0e5c7b18cd85befdcade817c8753062b3dff2 +size 1326061 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0177761c11259e135dafeb4218588dd4d436148c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/console.log b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/console.log new file mode 100644 index 0000000000000000000000000000000000000000..262e6cbcf5a328e16825d1d48e9d0d3f2f94c257 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/console.log @@ -0,0 +1,22 @@ +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 22:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 22:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 22:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 22:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/config.yaml +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.json +[02-20 22:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 23:11:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 23:11:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 23:11:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 23:11:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ebb2f3935ccb7e2bf4fd8c5bc8ca449cfa0b8d50 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/debug.log @@ -0,0 +1,353 @@ +[02-20 22:57:14|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 22:57:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 22:57:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 22:57:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 22:57:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 22:57:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 22:57:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 22:57:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 22:57:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 22:57:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 22:57:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 22:57:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 22:57:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 22:57:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 22:57:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 22:57:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 22:57:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 22:57:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 22:57:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 22:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 22:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 22:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 22:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 22:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 22:57:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 22:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 22:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 22:57:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 22:57:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 22:57:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 22:57:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 22:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 22:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 22:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 22:57:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 22:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 22:57:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 22:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 22:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 22:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 22:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 22:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 22:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 22:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 22:57:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 22:57:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 22:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 22:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 22:57:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 22:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 22:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 22:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.55 s +[02-20 22:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 22:58:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 22:58:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/config.yaml +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 22:58:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its orbit, returning to a position similar to the start but with a slight upward tilt. The office chair, still piled with pillows and blanket, subtly rotates on its own. All mandatory landmarks — TV on black stand, yellow leather sofa, TV displaying cityscape, black TV stand, carpeted floor, office chair with stacked pillows, wall-mounted pictures, yellow sofa with patterned pillow, floor lamp, office chair with pillows and blanket — are clearly visible in the final tranquil composition.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.json +[02-20 22:58:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 22:58:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.mp4 +[02-20 22:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 22:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 22:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 22:58:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 23:11:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 23:11:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 23:11:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 23:11:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 23:11:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 23:11:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..29d9c9e96982b92f0b23c9a07ffd8745df3794b0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle, wide shot centered on an office chair piled with pillows and a blanket. The chair has a black mesh back and silver base with wheels. To the left, a large TV on a black stand displays a cityscape. The floor is beige carpet. The camera begins a slow clockwise orbit around the chair.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8dea6d6a7e8c85e0662b995cf5db14239bf23536 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43cd21183730525ca03b2524b509486428a75cf0cbcc00bbe80a433caf4c26a8 +size 1356951 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..69a5ea8bb3bd12cdbc1e819aa7c05cd906f156fa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the orbit, the camera reveals a yellow leather sofa to the right, adorned with a patterned pillow. Wall-mounted pictures are visible behind the sofa. The TV still shows the cityscape. The camera movement is smooth, maintaining focus on the central chair while gradually exposing more of the room’s layout.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8d3208afaf7c5b18d7b7dd317679d0d1144319b1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c23f24393338134439192a0689ae01a8840ced72cc89f266a0b4d03b4942bf5 +size 1271291 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..ea7fe32a0b2702f0e8f87bf5ddee138e1c7532bb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera moves further around, showing a second yellow leather armchair near a side table. A floor lamp casts soft light in the corner. The beige carpet extends across the entire floor. The office chair remains piled with pillows and a blanket, and the TV continues to display the cityscape on its black stand.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2bec6e9e128180a53e0114a94ce666b692a20014 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1671654ce8dec018206079dcd8911ed2c4a519c699649b21061b3adb196d2b +size 1140896 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a3f5b25a0f8efd08d0b534a210531a33712e322c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: Now halfway through the orbit, the camera focuses more closely on the stacked pillows — one with a sunflower motif, another with circular patterns — and the light blue-gray blanket. The TV and black stand remain visible in the background. The camera tilts slightly downward, revealing the chair’s wheels on the carpet.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c1de50718d7a97d4f6c9658f3aa4835f9963227c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9be7d16914b5325ef2c7bf3c0240e4292b998858f5ed8bdc6b857c98c12d75a +size 1423892 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..8690ef8197aeaa693e714595539ea08a64df6a7a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera dips lower for a close-up of the chair’s base, showing the silver five-wheel mechanism rolling slightly on the beige carpet. A label is visible on the underside of the seat. The yellow leather sofa and patterned pillow reappear in the background as the camera begins to pull back slightly.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d0b3784b69199219d8cddd12de34aa20b2aabb0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:211f0fad510cf4543a3f96cd867bd8a45222789e355dc9fe46eccd4a4a3face3 +size 2196972 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..f0c9d0a5c4742a985d2158e6b4b6e3ba18d03ae5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: As the camera continues its clockwise orbit, it captures the rear view of the office chair, still laden with pillows and blanket. The TV screen, displaying the cityscape, comes into view again. The wall-mounted pictures and floor lamp are visible in the background, reinforcing the cozy, lived-in atmosphere of the room.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5ec4da7459e7b2da32694db05d2cf98821aece15 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19d367e30452bbeae7933e5dbd8da7a422be93ba67281c48d656f87f2535badd +size 2580984 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..e325dc6d074d6db764457be327232944d295e6dd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera moves toward the final quarter of the orbit, bringing the yellow leather sofa and patterned pillow back into prominence. The black TV stand and cityscape remain steady in the frame. The camera smoothly transitions, preparing to complete the full 360-degree view while keeping all elements in harmony.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..397de5ad8263c6337dea8efebce41ee1cb9217b9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:360778cdd1b698efefb9c3c4f8ce55822d2041f6b3129d1062228c56d97d9553 +size 1913487 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..f3b3670f0b038e94fb7d7f8505ef6f4494cb1857 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its orbit, returning to a position similar to the start but with a slight upward tilt. The office chair, still piled with pillows and blanket, subtly rotates on its own. All mandatory landmarks — TV on black stand, yellow leather sofa, TV displaying cityscape, black TV stand, carpeted floor, office chair with stacked pillows, wall-mounted pictures, yellow sofa with patterned pillow, floor lamp, office chair with pillows and blanket — are clearly visible in the final tranquil composition.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40044c172d722b99ba70284a0440dfa837c7b7e1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e902015e55fa3675466a9f115ae075f577b7c616a09a65dce8db53404479fe5 +size 1819171 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8a1b7bcac7880cf2bb2fd718fe2c8859a6ec4fa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/console.log b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..63ce15b37ff2369f8e01e680a3e67d0b5465a664 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/console.log @@ -0,0 +1,22 @@ +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/config.yaml +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.json +[02-19 22:42:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:55:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:55:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dcab9a8dc93b2d65d743a801f568be83ae5b675a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/debug.log @@ -0,0 +1,353 @@ +[02-19 22:41:44|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 22:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 22:41:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 22:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 22:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 22:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 22:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 22:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 22:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 22:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 22:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 22:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 22:41:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 22:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 22:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 22:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 22:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 22:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 22:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 22:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 22:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 22:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 22:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 22:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 22:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 22:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 22:41:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 22:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 22:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:41:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 22:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 22:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.02 s +[02-19 22:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 22:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 22:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 22:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:41:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 22:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 22:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.57 s +[02-19 22:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:42:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:42:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/config.yaml +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 22:42:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Finally, the camera pulls out to a medium shot encompassing the original dresser, its mirror, and both lamps. The reflection still includes the bed with the tufted headboard and blue painted walls. The broader context of the showroom is visible — symmetrical arrangements, multiple beds, and consistent overhead fluorescent lighting — concluding the video with a balanced, comprehensive view of the entire display.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.json +[02-19 22:42:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.mp4 +[02-19 22:42:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 22:42:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 22:42:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 22:42:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:55:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:55:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:55:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 22:55:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d8a036ad0591aa7009b1981abe878147e3c8e1c0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a wide-angle shot of a furniture showroom under bright overhead fluorescent lighting. Multiple beds and dressers are arranged along aisles. The camera begins panning right toward a central white dresser with an arched mirror and two white table lamps. Reflected beds and lamps are visible in the mirror, including a bed with a tufted headboard and blue painted walls in the background.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c62edef1ae5dbe010b66bce9d5ded837fbb91aaf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:300f34a79acd4e9e6ce80f82772d15902b72187fa07ace56c9b7b47afafb0867 +size 1626432 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..80debc57b1658ac1bf6f8eb47c1ad8b5cbe84ac0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"The camera continues its slow pan right, now fully framing the white dresser with mirror and lamps. The reflection shows more detail: the tufted headboard bed, blue painted walls, and additional lamps. The dresser’s ornate drawer pulls and paneling become more visible as the camera begins a gentle circular movement around its right side.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..905733276d475d37f31d410ce4cea1d6e1b9cde2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2016df13a39a371a92b45302d19f713b94fc6a37e2b8403a9d6446bbebdc2686 +size 1471937 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..5a3ca8fbbc3550ed88d278540551dce199f31732 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"As the camera circles the right side of the white dresser, it tilts slightly upward to emphasize the texture of the lamp bases and the curvature of the mirror frame. The reflected beds and lamps remain visible, reinforcing the depth of the showroom. Blue painted walls continue to appear in the mirror’s reflection, and overhead fluorescent lighting casts even illumination across all surfaces.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e531ebed878606398d0d374583f68b3af24e3370 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c91218fbc55f224a0b53c7008e7c94f1482b58955eaa33add57858d5896bfc28 +size 1220047 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..79e3237e8d7f00c37f23ccfc68c360a0171c3e9f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The camera completes its circle around the dresser and begins panning left, revealing a second identical white dresser with two lamps and a mirror. The reflection in this mirror again includes a bed with a tufted headboard and blue painted walls. The camera holds this composition briefly, showcasing symmetry between the two displays under the same overhead fluorescent lighting.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46a521836105e371faf10c8b88e51a09b728d143 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77b1d256292f076505f44a980b6f078692c4f22775d4fd5d3f1079a1dd100f19 +size 1328205 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..4648db7cb47a488bf30be6a4d4f3cc8836037704 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The camera pulls back slightly from the second dresser and begins tracking left along the main aisle. More bedroom sets come into view, including beds with tufted headboards and matching dressers. The blue painted walls line the back of the showroom, and overhead fluorescent lighting remains consistent, illuminating the entire space uniformly.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3dc3184cbd90ddb16da875e9e6750e9041144150 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d38943ba499efd312c5e33f4c9227df48768d8d7e3a760bae8d815de62ae78ca +size 993842 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..e2838231939d0e3911ad71dc042468c4e09b925d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Continuing its leftward track, the camera reveals the full length of the showroom aisle, showing multiple coordinated bedroom sets. Beds with tufted headboards are positioned against blue painted walls. White dressers with mirrors and lamps are spaced evenly, their reflections capturing the surrounding environment. Overhead fluorescent lighting ensures clarity and brightness throughout.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a373844b05b269503cff67a8406d7e25222b558 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b05c0bb7e8a1809d816bd00f71cc1d4bca1e11b3b77141b2f89f7fb025d74b3 +size 1433773 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..76592eadb8dc27e8bca55c16da54e575433b5e2f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"The camera slows its movement and begins zooming back in on the original white dresser with mirror and lamps. Focus shifts to the textured bases of the lamps and the clarity of the mirror’s reflection, which once again shows the bed with a tufted headboard and blue painted walls. The overhead fluorescent lighting highlights every detail without creating glare.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..238acb9d059b39dec9ba4559e32c7f716d861780 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:996f94ba96f303f040a7e43a2f484aea1154274b2430eeb240600f8e3f98bb97 +size 1262293 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..251b21c5cb9f8fe349e17d617480318dc70e0839 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Finally, the camera pulls out to a medium shot encompassing the original dresser, its mirror, and both lamps. The reflection still includes the bed with the tufted headboard and blue painted walls. The broader context of the showroom is visible — symmetrical arrangements, multiple beds, and consistent overhead fluorescent lighting — concluding the video with a balanced, comprehensive view of the entire display.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..02dfd592d3903b4ccf8e515e8a100ca2e2d7f530 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72160b4936ffa2d76b75f4dcaa2f033f6e23cc52f007b0c5fe4a1d7aa8086a04 +size 1032482 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6eac25ed24163c83390d780b54f569b2e24f971 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/console.log b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..4085dd4082d4e5103a9776630695a23876fc5242 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/console.log @@ -0,0 +1,22 @@ +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:17:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:17:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/config.yaml +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.json +[02-20 19:18:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 19:31:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 19:31:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 19:31:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 19:31:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f78a252b110d5d982d874970734ecb964407a30e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/debug.log @@ -0,0 +1,353 @@ +[02-20 19:17:18|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 19:17:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:17:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 19:17:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 19:17:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 19:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 19:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 19:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 19:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 19:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 19:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 19:17:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 19:17:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 19:17:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 19:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 19:17:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 19:17:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 19:17:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 19:17:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 19:17:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 19:17:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 19:17:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 19:17:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 19:17:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 19:17:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 19:17:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:17:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:17:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 19:17:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:17:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:17:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:17:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 19:17:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:17:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 19:17:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:17:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 19:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 19:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 19:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 19:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 19:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 19:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 19:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 19:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:17:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:17:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 19:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 19:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.25 s +[02-20 19:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:18:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:18:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 19:18:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/config.yaml +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 19:18:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Finally, the camera slows its rotation and zooms in toward the floral pillow on the backrest of the black office chair. It tilts upward along the chair’s back, highlighting the texture and design of the pillow. The yellow sofas, TV with cityscape, and maroon chair fade into soft focus as the video gently fades out, concluding the sequence.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.json +[02-20 19:18:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 19:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.mp4 +[02-20 19:18:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 19:18:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 19:18:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 19:18:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 19:31:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 19:31:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 19:31:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 19:31:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 19:31:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 19:31:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a5abf1bf2c83a4d2079972819d031e5960872a76 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a wide-angle view of a living room centered on a black office chair with a floral pillow. To the left, a large TV displays a cityscape; to the right, a yellow sofa with a patterned blanket. In the background, a maroon office chair is visible near a window. Beige carpet covers the floor. The camera begins a slow clockwise rotation around the chair, keeping all elements in frame.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ef63e9578c573d99e9506c0c1bdf18c4d0fa68c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff99c8a7c853289673f3889a0dfb789146019c42ce08dd8194dbdfd1fde9111d +size 1085859 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..ff61a4a3fbc70d2be10c70e16afeaf73f9af7a40 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing the clockwise rotation, the camera reveals more of the room’s layout: a second yellow sofa on the right, also covered with a patterned blanket, a floor lamp beside it, and framed pictures on the wall. The TV screen continues to display the cityscape. The black office chair remains the focal point, its position unchanged as the camera circles it.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0593335fa88c88dbb387d6f527c3fe8ac21ed2e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:754f52e1852a6e82ebcafa2c94fcc16235b7b40ecdd25a7a2acc38f98324b9dd +size 1079968 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..7a712feafa82c878ec440fc12337210f3377730e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"As the rotation progresses, the camera captures the full living room layout from a new angle — the maroon chair is now more prominent, and shelves with equipment are visible behind it. The TV screen still shows the cityscape. The camera maintains steady motion around the black office chair, ensuring continuity of scene composition and lighting.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d08567b5c5bdb0d4fe9ef7b60c7fc8084c9a25cd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:009e874928f7918287e99a159cb6dc6c693ba363d6c0aff14ec1264ad54b1827 +size 1160034 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..35425fa2b734ef73d1fa549aa92e18f4d6a6beae --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The camera tilts downward, focusing on the black five-star base of the office chair resting on the beige carpet. Close-up shots highlight the wheels and structural design of the base. The surrounding environment — yellow sofas, TV, and maroon chair — remain softly blurred in the background, preserving spatial context.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..10274b985f2a99d4d6aa5b65d2d0d9d7b5583be4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a88c5607e697adf2f502b4e36e40fce5665fad78f530e59c45de76c1344d4d59 +size 1274229 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ccdf946e7f80b743391a2d2cfe1d8c9b035a8006 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Moving upward from the base, the camera follows the chair’s central column, revealing the underside mechanics and the lower part of the floral pillow. The camera then shifts to a close-up of the pillow’s pattern: red flowers, yellow accents, green leaves, and gray swirls on white fabric, with visible stitching details. The background remains consistent with the living room setting.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e199dd0c02d22a7270704a2da4546572ccc601a0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf821a4cc9667171e180a02c04b38aeb95fd06b2418b37b921c9b47fccd1c122 +size 1416133 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..ebbfd20998c521a49684e266a93c451ce5cc45f7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Pulling back slightly, the camera frames the entire black office chair with the floral pillow from a slightly elevated angle. The yellow sofas, TV displaying the cityscape, and maroon chair are all visible in their respective positions. The camera begins another slow clockwise rotation around the chair, maintaining smooth motion and visual continuity.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6b1070e7dfd35597d86b318e4a1a734851b890d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3f95f329e791184186129c18becf5f53586d4fa7927ee711f2b91743dd4199c +size 1412549 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..7e7b2468dcada2cd9e709adfb583c129468d09fa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Midway through the second rotation, the camera offers a balanced view of the room — the TV on the left, the yellow sofa with blanket on the right, and the maroon chair in the rear. The black office chair with its floral pillow remains centered. The lighting is consistent, and the beige carpet provides a neutral foundation for the scene.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3ffc43c11c28726cf87661c0f648f8170f117b63 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:457c7cfcd7cbd62399ddfd9a0b09e47b10fff03f30cb16d0ccc793a1367061a6 +size 789068 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..6ebe0b699c26fe3339af3bf9691aded57d89989b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Finally, the camera slows its rotation and zooms in toward the floral pillow on the backrest of the black office chair. It tilts upward along the chair’s back, highlighting the texture and design of the pillow. The yellow sofas, TV with cityscape, and maroon chair fade into soft focus as the video gently fades out, concluding the sequence.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c43cd8fdbf79513ff29fe0a99944e6dea5848fa3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22f9979a19241e31b40cdcca9b2880120ac6fbcf2a3904ee254a48e536f13168 +size 1108047 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aae848c220b9e0b8d9dda22ca906eca41b24a747 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/console.log b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/console.log new file mode 100644 index 0000000000000000000000000000000000000000..22b1b3fb0742df5cfe0758014dc464587db7b6cb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/console.log @@ -0,0 +1,22 @@ +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:13:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:13:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/config.yaml +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.json +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:27:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:27:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:27:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:27:41|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..12802a0f5d14ce4f7ca06bf64e4aa291921d87cb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/debug.log @@ -0,0 +1,353 @@ +[02-20 04:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 04:13:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:13:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 04:13:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 04:13:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 04:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 04:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 04:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 04:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 04:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 04:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 04:13:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 04:13:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 04:13:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 04:13:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 04:13:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 04:13:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 04:13:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 04:13:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 04:13:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 04:13:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 04:13:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 04:13:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 04:13:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 04:13:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 04:13:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 04:13:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 04:13:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 04:13:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 04:13:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 04:13:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 04:13:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 04:13:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 04:13:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 04:13:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:13:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:13:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 04:13:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:13:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:13:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 04:13:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:13:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 04:13:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:13:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:13:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 04:13:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:13:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:14:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 04:14:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 04:14:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.19 s +[02-20 04:14:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:14:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:14:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/config.yaml +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 04:14:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Finally, the camera gently zooms in on the black metal bench, focusing on its intricate armrests and slatted back. The red-and-white tiled floor stretches out beneath it, while the glass skyscraper and stone building with statues form a layered backdrop. The lighting is bright and even, suggesting midday sun. The video ends with this intimate close-up, leaving the viewer with a sense of quiet urban beauty and architectural harmony.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.json +[02-20 04:14:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:14:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.mp4 +[02-20 04:14:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 04:14:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 04:14:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 04:14:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 04:27:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 04:27:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:27:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:27:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:27:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 04:27:41|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..494b00d5f55cf780354f8483d9b32ae01f0d232e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a static wide shot of a black metal bench centered on a red-and-white tiled plaza floor. Behind it, a modern glass skyscraper reflects the blue sky and surrounding buildings. To the left, a stone building with large columns and stone lion statues is partially visible. A balustrade railing runs along the edge of the plaza, separating it from the street below.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63cce8f3b0e36a67bae63c92968b0ab44887583e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03878039243bf228faf433862a9b1f13fc5c5ccb8df86f245d0a02aa58efa816 +size 767965 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..c78a7f5fc07fe8cda08ac487f2291b26c64ca0e9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"The camera begins a slow leftward pan, bringing the stone lion statues into full view atop their pedestals beside the grand stone building. The balustrade railing extends across the frame, and a classic streetlamp appears near the railing’s corner. The red-and-white tile pattern dominates the foreground, creating strong visual lines that guide the eye toward the architectural elements.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ec633f5d0b794c077fcaf3a9e6f501f96b9f2682 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80f00b4b1df757626692f2a7ddbaeafb884816842d9ce817f33c2915372aa12c +size 1151489 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..acc81f56fd92f1fa0cf9c76b7c83508fd2f676ec --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Continuing the leftward pan, the camera reveals more of the stone building’s façade and the full majesty of the lion statues. Reflections in the glass skyscraper shift subtly as the camera moves, showing distorted images of nearby buildings and clouds. The black metal bench remains centered in the lower third of the frame, serving as a visual anchor throughout the movement.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b63c594622be209f8292db44aee32ec92265951c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e5b13a1285791abbc961bbf607117c7c17041522010a2539b306cc5090ce2e2 +size 1080171 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..7f3092a5d06cb34d0d97e583ba0dcac5ee1de1f8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The camera now begins to arc around the back of the bench, shifting perspective to reveal the opposite side of the plaza. More benches and trash receptacles come into view, arranged symmetrically along the balustrade. The glass skyscraper looms overhead, its reflective surface capturing dynamic sky patterns. The stone building with statues is now visible in the background, maintaining continuity.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..32adb303655d315429289b4287e44206d8ab4d75 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d644cd7676e7ba8e3b0d0d78e6812388d44b6e769b541fa888012cef6386fa6f +size 984992 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..9dba7d2569c8b1c68ecb11cd253a0e06e80ff2df --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"As the camera continues its circular motion, it captures the interplay of light and shadow on the red-and-white tiled floor. The black metal bench is now viewed from behind, its slats and ornate legs clearly visible. Reflections in the glass building become more pronounced, showing the movement of clouds and the silhouette of distant towers. The balustrade railing curves gracefully along the plaza’s edge.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ff47cf0ce655b07754c00913679a4c3c755e3d62 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4002e15d0073b766716a8cf6a1332350d028d783f7118f1ef25824532701763d +size 1000162 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..a778f6fe58ec9ba767f3bf8a894503ab1882e452 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"The camera sweeps further right, completing more than half its circular path. The stone building with statues reappears in the distance, framed by the glass skyscraper’s reflection. A second streetlamp comes into view near the railing, echoing the first. The red-and-white tile pattern creates rhythmic diagonals across the frame, reinforcing the plaza’s geometric design.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..83a9502690a7f5c88cf06ba619d638ac5da374c9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5d32c8cbb36a7c372c1afa283ff51c799e836b3b6ccbff871f7322c99f8298a +size 1713378 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..ad462d6b46291f59ac634f454186c9f93d0d031f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Now nearing the end of its arc, the camera positions itself slightly to the right of the bench’s original front-facing angle. The black metal bench is again central, but now seen from a three-quarter rear perspective. The glass skyscraper fills the upper frame, its reflections shimmering with ambient light. The balustrade railing and stone lion statues remain visible in the periphery, ensuring visual cohesion.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2da7ff11a363508caa56e59ffb623a3bb60d1ba9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eccdcbee27066a6bc3af571736ab3d11fde0ce86bc2eb86f568ae5c298ad092f +size 1295442 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..af66966d16a1f44317c2b7616c54ba162c4c9115 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Finally, the camera gently zooms in on the black metal bench, focusing on its intricate armrests and slatted back. The red-and-white tiled floor stretches out beneath it, while the glass skyscraper and stone building with statues form a layered backdrop. The lighting is bright and even, suggesting midday sun. The video ends with this intimate close-up, leaving the viewer with a sense of quiet urban beauty and architectural harmony.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7dfcdb8adc72d8a5220804a83ad54b595395a3e8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1144070f6a6f9a3b0a27ee59467ea84cde5aceee58aa95fb31ec76fd1cf60b28 +size 1042725 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d4a568ce2c343ec13fc2c28de8bf1d54731d8c0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/console.log b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/console.log new file mode 100644 index 0000000000000000000000000000000000000000..4e1a5c0f1002cf66ed6b170d659cba591484426b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/console.log @@ -0,0 +1,22 @@ +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/config.yaml +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.json +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:40:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:40:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:40:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:40:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b717e5a0ec502e1aa8b042807f08fc6c49995384 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/debug.log @@ -0,0 +1,353 @@ +[02-20 10:26:25|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 10:26:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:26:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:26:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 10:26:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 10:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 10:26:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 10:26:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 10:26:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 10:26:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 10:26:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 10:26:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 10:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 10:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 10:26:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 10:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 10:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 10:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 10:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 10:26:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 10:26:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 10:26:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 10:26:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 10:26:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 10:26:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 10:26:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 10:26:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 10:26:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 10:26:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 10:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 10:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:26:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 10:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 10:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 10:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 10:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 10:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 10:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 10:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:26:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:27:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 10:27:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 10:27:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.77 s +[02-20 10:27:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:27:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:27:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/config.yaml +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 10:27:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its circular motion and settles into a stable wide shot that mirrors the initial composition. The round table with its black rotating center sits perfectly centered beneath the grand chandelier. The twelve patterned chair covers encircle the table symmetrically. The white wall with the yellow accent is visible on the left, and the beige curtains flow down the right wall. The lighting remains soft and even, enhancing the luxurious textures and colors. The video ends with this serene, fully composed view of the elegant dining room.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.json +[02-20 10:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.mp4 +[02-20 10:27:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 10:27:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 10:27:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 10:27:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 10:40:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 10:40:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:40:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:40:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:40:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 10:40:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..fcaccc3bae6fb3daef5fbe5906a58ad6aea43a7c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a wide, static shot of a luxurious dining room. At the center is a large round wooden table with a glossy black rotating surface, surrounded by twelve high-backed chairs covered in gray-and-white damask-patterned fabric. Above, a grand multi-tiered crystal chandelier emits warm light. To the left, a white textured wall features a horizontal yellow accent panel. To the right, beige curtains hang from floor to ceiling. The camera begins a slow clockwise pan, keeping the table and chandelier centered.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2294e5df6692e40b807aa4b489e77c6262c2a136 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0c23f1b5bfdebd1ec1e7afcd864a0a3718af47555b3404f28feb595e0142dc8 +size 1053132 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..fb2053bd26baea22645ec35a414ed9a55bbdfdd3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing from the first segment, the camera pans further clockwise, revealing more of the white wall with the yellow accent panel. The patterned chair covers remain prominent in the foreground. The place settings — white plates, silverware, and folded napkins — are clearly visible on the table. The chandelier’s crystals glint softly as the camera moves, and the recessed ceiling lights become more noticeable against the layered architectural details.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..82ebc842baa730f753fe40179e5807208bdf4722 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:400578f62e8756e417d2acbb6f447a326cbfdf8216ccf2a8ec3105fcba29fc94 +size 1033242 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..18f8134ef08f7b1fe3ad113a8f305f00b07e711c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera continues its clockwise orbit, now showing the rear-left quadrant of the room. The white wall with the yellow accent recedes slightly as the camera reveals more of the dark wood-framed doorway behind the table. The beige curtains on the right wall come into fuller view. The patterned chair covers maintain visual continuity as the camera glides smoothly, preserving the elegant symmetry of the dining setup.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..96128471f6b17f51ff55a83c280a02df29685cdb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df2bcf46a9ffb835c64d00edeff10540c99e51dc3b0a186f470cd523b91e8b84 +size 985783 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..55a422ca188ce458f50c81fbeb01e13d29350ec1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: Now at the midpoint of the pan, the camera faces the far side of the room, dominated by the beige curtains and dark wood trim. The chandelier remains visible overhead, casting reflections on the black table surface. The camera angle tilts slightly upward, emphasizing the ceiling’s geometric recessed panels and embedded spotlights. The patterned chair covers continue to frame the shot, their ornate designs contrasting with the minimalist walls.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58c70910ea685a07837c92a2c9fe0695a716becd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a81e37954d30d02a8f4058ff4f4fdfaa00f8f871dca202826921e935ea7bb005 +size 1112306 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..533463c724fa04ff2c5c021412ae75c2727adfd2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera continues its clockwise movement, now transitioning toward the rear-right quadrant. The beige curtains fill much of the background, while the edge of the white wall with the yellow accent begins to reappear on the left. The camera maintains a steady height, ensuring the table’s rotating black center and the place settings remain consistently framed. The chandelier’s sparkle becomes more pronounced as the camera angle catches its crystals from a new perspective.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f30535311d10f5dc4b8aad4ba5d264129cc76342 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:598cf183d264d927fdb81cdd45eef86c02364510a374761fa71edb5ca029f3ef +size 1264419 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..b12c7a725e20664cbd35f7abff5dbb134ae87585 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: Moving into the final third of the pan, the camera reveals the right-front portion of the room. The beige curtains still dominate the backdrop, but the white wall with the yellow accent is now clearly visible again on the left. The patterned chair covers are seen from a slightly elevated angle, highlighting their tailored fit and decorative motifs. The camera subtly zooms in to emphasize the texture of the table’s wood grain and the glossiness of its black center.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17dc8080468a92130d2517e0638cc8079b29be07 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:649e15c0823eb21f883ed8de42919bca732513d8d670fd5535d9fd10b8044a83 +size 2018959 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..bdd148917b4896fe38a25322e45316d11a6e9e68 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera continues its clockwise sweep, nearing the starting point. The white wall with the yellow accent panel is now fully in view on the left, mirroring the opening shot. The chandelier hangs centrally above the table, its crystals refracting light across the room. The camera begins to pull back slightly, preparing for a wider framing. The patterned chair covers, place settings, and polished floor tiles are all captured in crisp detail, reinforcing the room’s cohesive aesthetic.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9cc6aaf99f74088270baa919c110b204a120ea72 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3279885f76c201f6e90d017bbe8cf7537c6a9ac8a44d8cf61cc8fae782aa40c6 +size 2251937 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..f51916b0d73aed3893850a4d21e99fba3691efcf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its circular motion and settles into a stable wide shot that mirrors the initial composition. The round table with its black rotating center sits perfectly centered beneath the grand chandelier. The twelve patterned chair covers encircle the table symmetrically. The white wall with the yellow accent is visible on the left, and the beige curtains flow down the right wall. The lighting remains soft and even, enhancing the luxurious textures and colors. The video ends with this serene, fully composed view of the elegant dining room.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33824bc71862fe2c7139c0b03f1ce5bd364cee2e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:754861fe60dbbc30381963a6a687feece1fc885789b625b6dfa4cde29a88a639 +size 1596291 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8049fc01627dc8a563c8b8157e0ba1b8365894c1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/console.log b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ed0b67f45654772c6a90e2ff5c39b6d11af34828 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/console.log @@ -0,0 +1,22 @@ +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:42:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:42:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/config.yaml +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.json +[02-20 14:43:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:56:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:56:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:56:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:56:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0873dfbb98fa5531134a53766b3935a287fdbcf0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/debug.log @@ -0,0 +1,353 @@ +[02-20 14:42:30|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 14:42:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:42:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:42:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 14:42:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 14:42:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 14:42:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 14:42:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 14:42:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 14:42:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 14:42:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 14:42:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 14:42:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 14:42:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 14:42:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 14:42:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 14:42:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 14:42:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 14:42:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 14:42:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 14:42:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 14:42:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 14:42:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 14:42:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 14:42:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 14:42:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 14:42:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 14:42:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 14:42:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 14:42:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 14:42:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 14:42:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 14:42:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 14:42:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 14:42:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 14:42:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:42:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:42:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 14:42:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:42:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:42:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:42:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 14:42:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:42:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 14:42:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:42:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:42:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 14:42:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:42:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 14:42:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 14:42:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 14:42:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 14:42:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:42:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:42:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:42:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:43:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 14:43:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 14:43:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.85 s +[02-20 14:43:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:43:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:43:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 14:43:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/config.yaml +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 14:43:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its clockwise orbit, returning close to the starting viewpoint but slightly zoomed out. The red circular base, blue bicycle seats, and statue are all clearly visible. The pink building and arched metal gate are once again in their original positions relative to the ride. The surrounding trees and playground elements complete the tranquil, overcast-day scene.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.json +[02-20 14:43:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.mp4 +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 14:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 14:43:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 14:43:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 14:56:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 14:56:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:56:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:56:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:56:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 14:56:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3f03ea24f9261d46b86c7e2876438c7dc39edc5a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a wide view of a children’s playground. Centered is a circular ride with a red base and blue bicycle-shaped seats. A statue of a cartoon character with a hat and wand sits atop a wireframe dome. In the background, a pink building and an arched metal gate are partially visible among trees. The camera begins a slow clockwise pan.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ff086fc0661f9d96862b6c2fcdf3010d13083ebb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f40daeb9774fae5ab4a1f58b46330b60503142112e9d5e69d821dc91de8979e4 +size 1880823 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..95597cc11fec467abc69f7b4c42e146a34757a54 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing from the start, the camera pans slightly further right, bringing the pink building more into view behind the ride. The arched metal gate becomes more distinct. The red circular base and blue bicycle seats remain in focus, with the statue still visible above. Concrete pavement surrounds the ride’s elevated platform.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b6d91e03f425addf9bcb0372720f4a1c1e8d0ecb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcbb709f59e6d6c96a85b6ae8598f8fc8755bafa640ca27d5758608d6216bbec +size 1797898 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d984610814dd7b770aa91f1a69286d48a005acf3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera continues its clockwise movement, now showing more of the left side of the ride. The blue bicycle seat is prominently featured. Trees and distant playground equipment appear in the background. The pink building is temporarily obscured by foliage, but the arched gate remains visible on the far right.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5390d6ba170b8be26f4cf9730a956396ee38e22 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91775ce9b4a731a062f5e2044b9f3139688b0865a59eaeb4a944b4abc451afcb +size 1612687 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ee59fceed4ea07dc5ae32563c2aed74b60f586cf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: Midway through the rotation, the camera reveals benches and additional play structures behind the ride. The red circular base and blue bicycle seats stay in frame. The statue atop the ride remains fixed as the camera orbits. The pink building reappears briefly through gaps in the trees.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b0b76611b520bb9c1010a66a790a06cd7fdd186 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:889039cda13b286f3383ba4bc9cb64040945e55016e983e00d4f65cc37d2a0d0 +size 1551792 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..640422d8ff5df885aae1fe5f819461c6dadddbfa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera moves past the midpoint, now showing the rear of the ride. The arched metal gate is fully visible again, and the pink building appears behind it. The blue bicycle seats rotate into view from the opposite side. The wireframe dome and statue remain constant focal points.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e4a31b08feb465adeb82788e8d89ef00b9c50f6c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:829403b425c922c2640d2237784a2ec55303f1c049a45ccdc85b9907b293e3ae +size 1653678 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9c0bbca450d6ec874b00d22ba7528dd0cbd26105 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: As the camera continues its arc, the pink building becomes more prominent again, now seen directly behind the ride. The arched gate is partially hidden behind trees. The red circular base and blue bicycle seats maintain their position in the frame. Distant slides and climbing frames come into view.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7e53c39dcc69145cf76f8d63d199e2bec94a6c9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ea569802b64aa7316f8728d1847ff6d72647f5d09370a4863d042381de2cbdc +size 1861232 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..5a042f9dfa43128634f1465d1dcb121d754894cf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: Nearing completion of the circle, the camera shows the ride from a three-quarter rear angle. The statue atop the dome remains visible. The blue bicycle seats are clearly shown, and the red circular base anchors the composition. The pink building and arched gate are once again visible to the right, framed by trees.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb51c22011a0e9488b2a62f3e52a8cfdbc864ee9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81c3adf8ccef7a68c802e0aaf980cb7b85dddc0a73bdc5db624299720d3d2cc6 +size 1560761 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..107bc84f470ccd87c556b711c86feb10f52ec857 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its clockwise orbit, returning close to the starting viewpoint but slightly zoomed out. The red circular base, blue bicycle seats, and statue are all clearly visible. The pink building and arched metal gate are once again in their original positions relative to the ride. The surrounding trees and playground elements complete the tranquil, overcast-day scene.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9595c5428ceae5e78de3ec78773a34a8597934f6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71cd085b173be21130b093f3adb486f59404284e2f51d2a64a33765e8e1d819b +size 1691516 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fa9234635d842907ea843b92d8a5c359f15ba3e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/console.log b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/console.log new file mode 100644 index 0000000000000000000000000000000000000000..94d56e659859423dd7cf52e1ab6fc6ead8b2feb6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/console.log @@ -0,0 +1,22 @@ +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:08:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:08:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/config.yaml +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.json +[02-20 06:09:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:22:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:22:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:22:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:22:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1d962200f78bd90503810a7f98fc21ce91d29613 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/debug.log @@ -0,0 +1,353 @@ +[02-20 06:08:18|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 06:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:08:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:08:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 06:08:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 06:08:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 06:08:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 06:08:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 06:08:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 06:08:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 06:08:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 06:08:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 06:08:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 06:08:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 06:08:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 06:08:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 06:08:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 06:08:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 06:08:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 06:08:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 06:08:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 06:08:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 06:08:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 06:08:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 06:08:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 06:08:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 06:08:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 06:08:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 06:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 06:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 06:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 06:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 06:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 06:08:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 06:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 06:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 06:08:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 06:08:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:08:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 06:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:08:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 06:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:08:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 06:08:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:08:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 06:08:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 06:08:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 06:08:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 06:08:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:08:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:09:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 06:09:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 06:09:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.24 s +[02-20 06:09:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:09:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:09:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 06:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/config.yaml +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 06:09:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Finally, the camera frames the entire scene one last time: the spinning chair at center, the TV displaying the cityscape on its black stand, the beige sofa with blanket, the red office chair, the floor lamp, and the carpeted flooring. The black chair base and wheels are clearly visible, completing the visual narrative. The video ends with the chair still rotating gently, the fabric softly flowing, and all mandatory landmarks cohesively integrated into the frame.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.json +[02-20 06:09:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:09:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.mp4 +[02-20 06:09:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 06:09:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 06:09:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 06:09:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 06:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 06:22:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:22:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:22:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:22:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 06:22:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..5aa8badfdaeec94a0922f8a8ab53da86cb8e74c5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a low-angle shot of a black office chair spinning slowly on a beige carpeted floor. The chair is entirely wrapped in a patterned fabric with large circles in purple, beige, and white. In the background, a beige sectional sofa with a gray leaf-patterned blanket is visible, along with a red ergonomic office chair near a window with sheer curtains. The camera begins to orbit the chair clockwise.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6af55e8eb10081f960fb8efc98e56194aa116e5b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb02cc5b984c782a647cc88ccfea72ac9c83a00a6d168a3a3d7b17f33b43bb21 +size 1316380 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..21c05aad9b1a3a131bc813bd0e2bebd930c5e481 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the chair continues to rotate, the camera reveals more of the room: a large TV on a black TV stand displaying a bright cityscape. A tall floor lamp stands beside the sofa, illuminating the area. The camera tracks the chair’s motion closely, maintaining focus on the swirling fabric while keeping the red office chair and TV in the background.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9beeb8e4067c7415be3c2ac7b3e2542e7e276712 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:803fd09e963608cf44f8b7218563c5b4197a2f45feeb2e1faa6df3d67627628b +size 1724263 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..b3725decc25bfbca9b7fb7e682f718ca39ba7498 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera tilts downward, showing the black five-star base and caster wheels of the office chair gliding smoothly over the carpeted flooring. It then rises slightly to capture the TV screen’s cityscape again, emphasizing the contrast between the dynamic chair and the static urban image. The beige sectional sofa and red ergonomic chair remain anchored in the frame.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..358f07d27e58e491acb68828f3d29d6d6b49e18d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b705f510145f3cd2ae1a5204af96c8cc43e066990f6097d6f5bfa952180a938a +size 1548178 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..8430aeb07115603b29ebba75ee09fbe561c531e4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Continuing its orbit, the camera pans left to reveal a small side table beside the sofa and framed pictures hanging on the wall. The patterned fabric covering the chair billows slightly with each rotation. The black TV stand and its contents — including speakers and media devices — come into clearer view as the camera circles.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..35cbbbba456387534dbd1a44f87b914eb62a389c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:661f80ed7f4fa4fd5b0dd302dc37441945acd76f60f1c4421a7e087b7c2c2166 +size 1469688 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..8e90693c1a6dcb0498ddc305b1dd6769e2de993b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The camera moves closer to the chair, zooming in on the texture of the fabric as it wraps around the chair’s contours. The rotation slows slightly, allowing the viewer to appreciate the interplay of colors and shapes in the pattern. The red office chair and beige sofa remain visible in the periphery, grounding the scene.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..398824a4ce9ecd1c0fce207311f2f58918416e43 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d1dc74a7c07eb51c5fde71fa11484f22f4c93eb1f33850e3fd43d82efa3433a +size 1212389 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..392ce93fdbf8fda428bb2c45b30f4ebbc25171ce --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Pulling back slightly, the camera captures a wider view of the living room, showing the spatial relationship between the spinning chair, the TV on its black stand, and the surrounding furniture. The floor lamp casts a warm glow, complementing the ambient light from the TV screen. The carpeted flooring extends throughout, adding to the room’s comfort.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df1596cc82ca71a2ecd64324995998103eeaa597 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f90c5edb742abcc5326e10e72febd3d37562558b7925bc2ddf2e8410a6c1abe +size 1184246 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..f3e9bd81f96c33fc4070cd5ef502439d76395b37 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"The camera now focuses on the chair’s base and wheels again, emphasizing their smooth movement across the carpet. The patterned fabric continues to sway gracefully as the chair rotates. In the background, the red ergonomic chair and beige sectional sofa with its leaf-patterned blanket provide visual balance to the composition.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..82dc23552d59737a8b4bc6583d81635ffa86496b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c16019fb7f71cfb96bd8eb2b4ac4332e050c954e6160e6f18047ce6ea3ac3cab +size 2080409 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..7c202631c409c10020a2a4cd43abcc2cd80a5582 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Finally, the camera frames the entire scene one last time: the spinning chair at center, the TV displaying the cityscape on its black stand, the beige sofa with blanket, the red office chair, the floor lamp, and the carpeted flooring. The black chair base and wheels are clearly visible, completing the visual narrative. The video ends with the chair still rotating gently, the fabric softly flowing, and all mandatory landmarks cohesively integrated into the frame.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7528b7c76f54f6b70d407989d1f8b12c4ab979bf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:845e324fbb9e4e010664ef202b3f03642d7b931e0972295ad681df51d5176ca4 +size 1428048 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5005218e080b1019e97f41a45d47e43fa00f16f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/console.log b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/console.log new file mode 100644 index 0000000000000000000000000000000000000000..d370b663c0dbc919afffe28eed78ddfc7cc18111 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/console.log @@ -0,0 +1,22 @@ +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:06:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:06:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/config.yaml +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.json +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:20:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:20:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:20:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:20:14|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..891cb758e1d4e07fc858011e22cc09eda78f89f0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/debug.log @@ -0,0 +1,353 @@ +[02-20 01:06:03|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 01:06:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:06:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:06:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 01:06:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 01:06:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 01:06:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 01:06:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 01:06:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 01:06:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 01:06:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 01:06:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 01:06:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 01:06:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 01:06:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 01:06:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 01:06:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 01:06:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 01:06:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 01:06:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 01:06:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 01:06:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 01:06:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 01:06:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 01:06:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 01:06:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 01:06:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 01:06:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 01:06:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 01:06:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 01:06:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 01:06:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 01:06:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 01:06:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 01:06:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 01:06:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 01:06:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:06:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:06:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 01:06:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:06:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:06:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:06:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 01:06:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:06:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 01:06:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:06:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:06:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 01:06:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:06:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.00 s +[02-20 01:06:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 01:06:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 01:06:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 01:06:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:06:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 01:06:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:06:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 01:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 01:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.49 s +[02-20 01:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:06:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:06:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/config.yaml +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 01:06:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera holds its upward gaze on the towering facade of illuminated glass windows stretching into the night. The concrete ceiling and recessed lights are barely visible at the very bottom. The scene ends with a lingering view of the glowing grid, evoking a sense of urban solitude and architectural scale.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.json +[02-20 01:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.mp4 +[02-20 01:06:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 01:06:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 01:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 01:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 01:20:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 01:20:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:20:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:20:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 01:20:14|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..39058d3b2c296ceaefbcfeeeadfac733ed199685 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle shot moving forward through a narrow alley at night. A concrete ceiling with evenly spaced recessed lights is visible above, casting soft illumination. To the right, a parked white car is partially lit under an overhang. In the distance, illuminated glass windows glow warmly from upper floors.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8c81e2b5ec472041caa86d995510593e08fcfa2d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea04bf5d13758e0fdf5398a78a46cab9997b0462ca44e983dd2150abf19eac55 +size 923099 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..c71fc8f3a45778ce4abd2466c6902511f339c93d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing forward, the camera begins a gentle upward tilt. The concrete ceiling remains prominent, with recessed lights creating circular pools of brightness. The parked white car stays visible on the right, while the illuminated glass windows grow larger in the frame, revealing interior details like furniture silhouettes.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8e4658ddb5f0afc4bdb342c975d35ca6ab86e67a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c56e897418aa05a0fd07d0509086749ede5284d96d4ea9cbea32883c1471c1a7 +size 833953 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..206c1716c87413e497e89d4b1503eff53011ac4e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera ascends further, now focusing more on the illuminated glass windows spanning multiple floors. The concrete ceiling is still present at the bottom edge. The warm interior lighting contrasts with the dark exterior, and the grid-like arrangement of windows becomes more defined.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7727d1f6911bc429103e4c777490d9f2500fa93c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09c13984e5cad01bd32837d14ce08ba4c442e95c7c2addc200ff8e7bc091892b +size 1600973 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..68511b761e4ca11d77571f2ca57f31b1f608f364 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera continues its upward motion, now centered on the glowing windows. The parked white car is no longer in view. The concrete ceiling with recessed lights remains at the lower edge of the frame. The windows reveal more interior detail, including ceiling fixtures and occasional shadows.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79d4cfe53e36eaa4c2233b6f62944c5e7ceb96f1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4737e0c216ba95c2bf32a5af99e3fb415c953970281fbb429ff470f56d1387a1 +size 1748901 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e948d76db9c211e76c9d580247102051401526c6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera briefly pans left, revealing a dark brick wall adjacent to the alley. It then returns to center, maintaining its upward trajectory. The illuminated glass windows continue to dominate the upper frame, while the concrete ceiling with recessed lights occupies the lower portion.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58ce32d68a7f14e5bc8988fd97a92a719d90750f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fecfeefb56dc2ca0fd125479e37d2d24b6a684a5b4993acf5f185f59f3519db +size 1642552 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..ecfba5612cbb682b23fbbc4d37c1f7d97ce2f46d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera passes beneath a large, flat concrete slab, momentarily plunging the scene into near darkness. As it emerges, the illuminated glass windows reappear, glowing even brighter against the night. The concrete ceiling with recessed lights is once again visible at the bottom.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6db7dd28d9aab6867b92cdb703746766f2b834ee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f06651d3c42c598281ef0a89bf09e469ad29c7eeb794bfe4b4ae40af2b220e8 +size 1758445 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..f65c4d1621c06c8d42fcfb8460b57e643db73126 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera continues its steady ascent, now almost entirely focused on the vertical array of illuminated glass windows. The concrete ceiling and recessed lights occupy only a thin strip at the bottom of the frame. The windows’ warm glows create a rhythmic pattern against the dark sky.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ceba4a52ab6275e5a650d60921a6d7006b4a0cd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:554ce4cacbb760eabddfc81f2df02541e417fa1511ce02a1b27dde54461c416e +size 2573154 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..5de89ed92f1415481204572cfa3bb32ca563dd72 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera holds its upward gaze on the towering facade of illuminated glass windows stretching into the night. The concrete ceiling and recessed lights are barely visible at the very bottom. The scene ends with a lingering view of the glowing grid, evoking a sense of urban solitude and architectural scale.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5ead4f5389be03126d5fe0446e60a341e11ca38b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d1bb8930719dc23b510f1df8b53d67f6d8e81018418087894abd7b4a0e28347 +size 2033066 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbf192009d956777370768faafe56e8cb077afca --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/console.log b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/console.log new file mode 100644 index 0000000000000000000000000000000000000000..615333c3a09b079a983207a52395f8b9a920bf9f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/console.log @@ -0,0 +1,22 @@ +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:46:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:46:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/config.yaml +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.json +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:00:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:00:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:00:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:00:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..147ad7a330631f63a6f675d62152b5e8b19cafd1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/debug.log @@ -0,0 +1,353 @@ +[02-20 11:46:21|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 11:46:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:46:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:46:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 11:46:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 11:46:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 11:46:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 11:46:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 11:46:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 11:46:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 11:46:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 11:46:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 11:46:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 11:46:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 11:46:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 11:46:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 11:46:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 11:46:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 11:46:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 11:46:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 11:46:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 11:46:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 11:46:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 11:46:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 11:46:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 11:46:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 11:46:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 11:46:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 11:46:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 11:46:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:46:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:46:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:46:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 11:46:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:46:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 11:46:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:46:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 11:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-20 11:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 11:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 11:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 11:46:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:46:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 11:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:46:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:47:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 11:47:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 11:47:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.70 s +[02-20 11:47:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:47:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:47:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/config.yaml +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 11:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: The camera reaches the bottom of the stairs and continues onto the wooden walkway flanked by grassy slopes. It moves backward slightly, offering a final wide-angle view of the entire sequence: the stairs, the circular sculpture atop the platform, the paved brick area, and the black metal railings — all under the same soft, overcast lighting.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.json +[02-20 11:47:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.mp4 +[02-20 11:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 11:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 11:47:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 11:47:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 12:00:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 12:00:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:00:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:00:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:00:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 12:00:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0882805f0232fb30cb9d6d4ad76edbb265cb1dc0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a forward-moving shot along a straight wooden walkway bordered by grassy slopes on both sides. Fallen leaves scatter lightly across the planks. In the distance, a set of wide wooden stairs rises between tree-lined embankments under an overcast sky.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9c6132ea729dee3eb2dc7fb488c4129dc775b2b3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05a8a45252445818b0e9226c64de2e3d16a9ac33fccc8fa4b73685b145f77b72 +size 1567698 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2aacadc41e049a446086dd6f63e32a534919af58 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera advances closer to the base of the wooden stairs, revealing metal railings on both sides. The stairs are constructed of dark brown composite material, with scattered yellow and brown leaves resting on the treads. Tree trunks flank the entrance, framing the ascent.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d7a6b0bbae3622a8934ea3d42d7869553444e8bf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b984dc56810bdb76321d563f5554215802d2321fc914f6fa4b68df5952634591 +size 1872842 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..42ef04a785ee704eef0a5f42e785d0bfd9a10703 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera begins climbing the stairs, moving steadily upward. Each step is visible, with leaves accumulating in corners and along edges. The black metal railings remain prominent on either side, and the overcast sky is visible above through gaps in the canopy.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0b84e4e9c974934d1484dcd7bfadaa49aafe68b8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d3f9903fcb5318278160930f9139e11f568384f59467117e963b5a383063dee +size 2020566 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9e5a15d38726a5acd02d6cb286356c0514aee4c8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: Reaching the top landing, the camera transitions onto a paved brick area surrounded by black metal railings. A circular sculpture mounted on a low pedestal comes into view, centered within the platform and partially framed by tree trunks. The ground is clean but shows signs of seasonal leaf fall.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c4e1a8df8e91a5652016530889cf67c7455e3788 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64762fe6f63421f81c1f33638635610937b5e540f8ca02e05678bc4a36252e3a +size 1554057 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e0aec4ebf7d9c34f67319f40c2ae536a34dc6db0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera pans slightly to the right, bringing the circular sculpture fully into focus. Its metallic, ring-like structure contrasts with the surrounding greenery. Behind it, the open landscape stretches into the distance, with trees and a faint horizon visible under the gray sky.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62a3b0cb3410f3d393e26c17daa6f02f73281c2c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21f746bdad081437279806f7f6cd16b43a566521011bdb08c85483ba9490c230 +size 1814870 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..879b7ac597878b253d9a7304a49b9ae6b56769ff --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera begins to turn around, pivoting to face back down the stairs. The paved brick area remains in view momentarily, with the sculpture now behind the camera. The black metal railings guide the eye downward as the descent begins.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..716912f2f04212aca51033394536c97434af31b0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0671ab09f0aab652ae901ea6403bf55c9c3eb76f70c00d06c2f247faa5efecd +size 1821169 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..101832ae189de74e333bbc970d2398b8a3567cff --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: Descending the wooden stairs, the camera retraces its earlier path. Leaves are still visible on the treads, and the metal railings on both sides frame the descent. The tree-lined path below becomes clearer as the camera lowers in elevation.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57473e9d692e64c0b3dfd4c857cec61d76fcbb11 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5a6ebd6e51e5c815dd56b33abb9e6a20875d2e1dcd2a422d022f1143450b059 +size 1430951 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..09c9a72277e7c77d241f1adb4cc855caaca39ae8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: The camera reaches the bottom of the stairs and continues onto the wooden walkway flanked by grassy slopes. It moves backward slightly, offering a final wide-angle view of the entire sequence: the stairs, the circular sculpture atop the platform, the paved brick area, and the black metal railings — all under the same soft, overcast lighting.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2fa774acacbecf2a872dc5530498ae92d545ac0c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:173be287f29464251f7113d5469272d754653a2a1272155b6de0b533ec87d213 +size 1709902 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8712e655c01e0386e1c284f0173d735b5cd0393 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/console.log b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/console.log new file mode 100644 index 0000000000000000000000000000000000000000..cff47ac8d68694cb2210f9c16e5183ee04b78e49 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/console.log @@ -0,0 +1,22 @@ +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:58:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:58:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/config.yaml +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.json +[02-19 21:59:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:12:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:12:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:12:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:12:49|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6a62823485ffcdcd244bb26b0d4cc80cc428ec73 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/debug.log @@ -0,0 +1,353 @@ +[02-19 21:58:40|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 21:58:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:58:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:58:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 21:58:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 21:58:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 21:58:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 21:58:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 21:58:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 21:58:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 21:58:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 21:58:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 21:58:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 21:58:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 21:58:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 21:58:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 21:58:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 21:58:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 21:58:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 21:58:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 21:58:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 21:58:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 21:58:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 21:58:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 21:58:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 21:58:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 21:58:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 21:58:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 21:58:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 21:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 21:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 21:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 21:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 21:58:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 21:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 21:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 21:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:58:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:58:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 21:58:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:58:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 21:58:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:58:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:58:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 21:58:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:58:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:59:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 21:59:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 21:59:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.57 s +[02-19 21:59:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:59:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:59:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 21:59:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/config.yaml +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera completes its 360-degree rotation, returning to face the Macy’s storefront directly. All mandatory landmarks — orange armchairs, round coffee tables, black rug, digital kiosk, Coca-Cola vending machine, and The Children’s Place store — have been captured in sequence during the pan. The video ends precisely where it began, with the same composition and lighting.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.json +[02-19 21:59:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.mp4 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 21:59:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 21:59:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 21:59:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 22:12:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 22:12:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:12:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:12:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:12:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 22:12:49|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3e6c9d33f3004c850a544b3c8233822bd4a42430 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a static wide shot of an empty mall corridor. Centered in the frame is a seating area with four orange armchairs and two round black coffee tables on a black rug. Behind them, the Macy’s storefront glows with its red star logo. The white tiled floor stretches outward under bright overhead lighting.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..47d61dfc246458cc3b90bdbd1a2a2784032519eb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6d107f04a5cfbc55ade99715a0ca91734af954ddadd61997d9eca84c293ba75 +size 704618 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..f48c8498d2377f0a60858393d612f694fef07794 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera begins a slow clockwise pan. The Macy’s storefront remains prominent as the left edge of the frame reveals a silver digital kiosk standing beside the seating area. The orange armchairs and black coffee tables remain fixed in the center, while the background begins to shift slightly.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c9d4ba33dfef100e38c51ba7a57e752c94ff370c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:573a9a5d3ab6e156b3920482721b42c23c77118c526f6539e37be1a40b977f85 +size 1242956 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..44578fb7ec9a62a7c4e28a077629c71b32a87743 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: Continuing the clockwise pan, the digital kiosk moves toward the center-left of the frame. The Macy’s storefront exits the right side of the view. A distant store entrance with yellow trim becomes visible in the background, and the white tiled floor pattern remains consistent.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9cf90a91a544ba69b1723ff4677af142b92ec52 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13f4d0001c95e73be7b7953308bc96e5e1a5cf520062d0e5b7b4fa84518d46dd +size 993775 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..cf17b8a43b785b9d4cf19d10894a1c53ad320e9a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera pans further right, now showing a Coca-Cola vending machine in the mid-background near the yellow-trimmed store entrance. The orange armchairs and black coffee tables still dominate the foreground, and the black rug beneath them remains clearly visible.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e09c116ce9a6a49dab826a58f355159d7d88dc35 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fca3b3b36c98487eabb96af34696ebedcf9b3244b4f3b6160472e69807f636da +size 1126387 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..9022e49b3d721a828ecb42a25a428165c3cbd38c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The Coca-Cola vending machine is now centered in the background as the camera continues its rotation. Above it, the sign for 'The Children’s Place' store comes into view, hanging from the ceiling. The lighting remains uniformly bright, and the tiled floor extends continuously.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d91cd9b90f86a998b8c90f53cc298cae29cae1d8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7687d2485472915da484744c147944414f3e878050d0c503cef01d045e98025 +size 1129134 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..92c357c89c51959eccc895243fa398669206dce6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The Children’s Place store sign becomes more prominent as the camera pans past it. The orange armchairs and black coffee tables remain anchored in the foreground, rotating with the camera’s perspective. The black rug beneath them stays clearly defined against the white tiles.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d4695314a4ba365550a666ad3693ea562a5f4762 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6729a709eba662fbf5f684dd55442cc96c35f89fef00d874899001f456569c90 +size 1043965 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..773be1185c440e3b56854d2f3b87404a4fa8b7e0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera continues its clockwise arc, now beginning to reveal the digital kiosk again on the far left. The Macy’s storefront starts to reappear at the far right edge of the frame. The scene remains devoid of people, maintaining a quiet, sterile atmosphere.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7b53540b2b0a2e89d46a5b1b8b805f1daee6505 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e20a54ec8c22f3cc46b40fb890c93c99227b17a445ffe853790f64eba7ed8f0 +size 1293970 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..e5ad8abe7ea54bb9d8ec5e760c36d187800a557e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera completes its 360-degree rotation, returning to face the Macy’s storefront directly. All mandatory landmarks — orange armchairs, round coffee tables, black rug, digital kiosk, Coca-Cola vending machine, and The Children’s Place store — have been captured in sequence during the pan. The video ends precisely where it began, with the same composition and lighting.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f828406ba47ef30ace617f81aaada4aeb06d25ca --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2e17f248fed7a1eb3aa55444b0c926496aabad9fdb41b89d055214de470f62e +size 1528149 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b37aefa442acee257cd849083a9347aa285c854a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/console.log b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/console.log new file mode 100644 index 0000000000000000000000000000000000000000..d6e0837080657fa0e6d0d744a3fa2f00611666dd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/console.log @@ -0,0 +1,22 @@ +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/config.yaml +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.json +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:45:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:45:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:45:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:45:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3f1bea83bda0f47e804dd31ff89ba3568bc57680 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/debug.log @@ -0,0 +1,353 @@ +[02-20 08:31:39|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 08:31:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:31:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:31:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 08:31:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 08:31:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 08:31:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 08:31:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 08:31:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 08:31:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 08:31:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 08:31:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 08:31:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 08:31:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 08:31:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 08:31:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 08:31:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 08:31:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 08:31:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 08:31:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 08:31:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 08:31:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 08:31:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 08:31:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 08:31:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 08:31:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 08:31:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:31:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:31:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 08:31:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 08:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:31:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 08:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:31:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 08:31:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:31:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 08:31:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 08:31:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 08:31:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 08:31:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:31:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 08:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 08:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.55 s +[02-20 08:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:32:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:32:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/config.yaml +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 08:32:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera pans downward toward the black five-star base of the central office chair, emphasizing its movement across the carpet. It then tilts up one last time to frame the entire room — ensuring all mandatory landmarks are visible: red office chair, TV displaying cityscape, stacked patterned pillows, blue exercise ball, side table with remote, beige leather sofa, black wheeled base of chair, carpet texture, central office chair with pillows, wall art, and floor lamp — concluding the video with a dynamic, immersive sweep of the space.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.json +[02-20 08:32:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:32:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.mp4 +[02-20 08:32:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 08:32:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 08:32:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 08:32:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 08:45:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 08:45:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:45:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:45:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:45:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 08:45:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c61ad8f2f62fc6295fb357313f0caf85adb886a6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video starts with a low-angle shot focused on a black office chair centered in a living room, piled with stacked patterned pillows — one with large purple and beige circles, another with white background and floral design — and a light blue blanket draped over its side. To the left, a beige leather sofa with matching throw pillows is partially visible. The camera begins a slow clockwise rotation.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c752f4960e75ebd1dc7a2b907595029e140199c1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee5f0471862ec3debe2fe0605ccaaf4ef9a1ce4768aa1011a51210ae8aebba71 +size 1274909 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..835f75f5fcd9f3325005fff0adc8dd5d1142d960 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the rotation, the camera reveals a large flat-screen TV displaying a vibrant cityscape with skyscrapers. Behind the TV, a red ergonomic office chair is visible on a black media console. The beige leather sofa remains in view, along with wall art — framed photos and prints — hanging above it. The plush cream carpet covers the floor beneath.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..180b75587425dd4a43c63ecb75fcfc5f85f61f15 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a2149b45ecd7d45d2594e285c2cd0dbad0fbbe20c3dcd816fbe70678dfbce37 +size 1808215 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..28f1e87b1a485741df6408a49d696dfd39bdd686 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera continues its clockwise orbit, now showing a tall floor lamp with a warm glow near the far wall. The red office chair is more prominently framed. The camera tilts slightly upward, capturing the ceiling and upper wall decor, then begins to dip lower toward the base of the central chair.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b65a2340592eb14cf6ab2043b66dadd37acdd6d7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae1c1515245060ac9d3f2886f3af987b057f0f41b4a5d1517479199ee614d964 +size 789725 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a4d3c675d16d8967fabd15819cf03f8db447e8a7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera lowers significantly, focusing on the black wheeled base of the central office chair as it rolls slightly forward across the textured cream carpet. The five-star base and casters are clearly visible. The beige sofa and side table with a remote control appear briefly at the edge of the frame.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..27b051efdf4c91adceb97b73938b50a315fc62f8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a3c30002d3410d768e6688e0f1d71ce703c0cef5580830e6cf03c44f588a035 +size 1430529 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c21a421c53c76b2bb5ba2ee764f729c0df78f18c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: Rising again, the camera captures the full height of the central chair with its stacked pillows. A blue exercise ball is now visible tucked beside the beige sofa on the right. The side table with the remote control is clearly shown next to the sofa. The camera continues its smooth clockwise rotation.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24c3d93a100cff0a0ceb166cb6edd954e9c94d1f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12a314fc808555030b51033b50c3c5631c3dd9ffd04068829d51a0e80518165c +size 1642950 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..b031d6d23f52e7e1a60514201447bcc4ba281926 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera passes the red office chair and TV again, completing nearly half of its 360-degree rotation. The lighting highlights the fabric textures of the pillows and the sheen of the leather sofa. The camera maintains steady motion, keeping the central chair as the anchor point.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a162bd4b1baf6d8c94df4a1b692c63bca3e4e020 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:189310a608f243058834d416e3bba576be573078ce6604ca0fcc8592076ea670 +size 1002255 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..01223d2bde3f4a9f645d24921b93a5fefc8b0a44 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera completes its full rotation, returning to the initial viewpoint but now with slight positional changes — the chair has moved forward slightly. It zooms in briefly on the pillow patterns, then pulls back for a wide-angle view that includes the second beige armchair, the window with sheer curtains, and the ceiling fan above.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ee2c4c090ba4f5c567cedb2d0207a9508a72e2f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:464987cc896090eab087007da1862c1162ceeeead0ed41c166c029c31d36401a +size 1382472 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..9884269024467074e127605c21d276df9892144c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera pans downward toward the black five-star base of the central office chair, emphasizing its movement across the carpet. It then tilts up one last time to frame the entire room — ensuring all mandatory landmarks are visible: red office chair, TV displaying cityscape, stacked patterned pillows, blue exercise ball, side table with remote, beige leather sofa, black wheeled base of chair, carpet texture, central office chair with pillows, wall art, and floor lamp — concluding the video with a dynamic, immersive sweep of the space.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4a76026f931ee3414ffb97fc1a577928288e576a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6381081f9dfe909254fb014979484a1ef1f0ca2ed26195fb8fdbc9f1bb9ebe2b +size 1565475 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eaabacdb89f51e8222d737f21daa0c6a1d970f46 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/console.log b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/console.log new file mode 100644 index 0000000000000000000000000000000000000000..a997fe0dbfbdb2b13b79e21949be48acc7d96ca8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/console.log @@ -0,0 +1,22 @@ +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:18:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:18:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/config.yaml +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.json +[02-20 21:19:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:32:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:32:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..424878bc4e878783bfa0b1ebb751af63cf269c95 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/debug.log @@ -0,0 +1,353 @@ +[02-20 21:18:43|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 21:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 21:18:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 21:18:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 21:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 21:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 21:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 21:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 21:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 21:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 21:18:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 21:18:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 21:18:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 21:18:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 21:18:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 21:18:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 21:18:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 21:18:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 21:18:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 21:18:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 21:18:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 21:18:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 21:18:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 21:18:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 21:18:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:18:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:18:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 21:18:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:18:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:18:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:18:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 21:18:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:18:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 21:18:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:18:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:18:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 21:18:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:18:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 21:18:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 21:18:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 21:18:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 21:18:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:18:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:18:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:19:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:19:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 21:19:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 21:19:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.97 s +[02-20 21:19:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:19:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:19:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 21:19:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/config.yaml +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 21:19:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Finally, the camera pulls back for a wide-angle shot that encompasses the entire restroom. All mandatory landmarks are clearly visible: gray toilet stalls, white sink counter, wall-mounted mirrors, mirror above sink, reflection of soap dispenser, reflection of toilet stall, blue bucket under sink, and trash can with white liner. The video ends on this comprehensive, static overview, capturing the symmetry, cleanliness, and functional design of the space.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.json +[02-20 21:19:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.mp4 +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 21:19:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 21:19:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 21:19:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 21:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 21:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:32:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:32:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 21:32:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb0767fc135d010811be2a89de1c7636fbf95d9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a slow pan across a row of gray toilet stalls in a clean, well-lit public restroom. White porcelain toilets are visible inside each stall, and the speckled gray floor extends throughout the space. The camera moves slightly forward, offering a close-up of the stall doors and hinges, emphasizing their industrial-grade construction and neutral color scheme.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38205d946bed5dd4d7f4c4156d9a9b9a7bdc575c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:298ee7e15132c5d36428ad981733da0732ec4faced5a426c75f8521daf219e2b +size 2066239 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..bffe302b7c42d1220a082f17f0a20c3ddf8659f9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the stalls, the camera turns left to reveal a long white sink counter supported by black metal legs. Wall-mounted mirrors are positioned above each sink, and soap dispensers are installed beside them. The camera focuses briefly on the first sink and mirror, capturing the reflection of the soap dispenser in the mirror’s surface.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ac80066b88e7ab7af7b4ee834bd9538dbc9300f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78b85530f9eb5bb262b8ca0a52187faac02eae28e4b87645d43892fb5aa8bdd3 +size 1290367 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..6d7156a88906cd014a4f1706a577dbcfd8a0d78a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera moves along the sink counter, revealing additional mirrors and sinks. A blue bucket is visible under the counter, tucked neatly beneath the second sink. The camera angle tilts slightly downward to emphasize the bucket’s placement and the clean lines of the plumbing underneath the counter.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c25d22b01b2d6169d1c0e7f04ba92dc901df7623 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:200461d865908fcde000095af497c442f710e824444a55507fd4afbbde4bc155 +size 1149944 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..77e77b294b229b07b3b5f29fb7873190e240155f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the camera continues its path, it captures the reflection of a toilet stall in the mirror above the third sink. The reflection shows the gray partition and part of the white toilet, creating a layered visual effect. A trash can with a white liner is positioned beside the sinks, and the camera lingers on it briefly to highlight its presence.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6b83581dd186e48e974593e0e8e382367b5574c3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92cf123c73d91d9914a940c3526a8a454f4c9902491f6896243b35436774d1b4 +size 1052673 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..a45e06da7f3f583857063777405cc595d2026635 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The camera pans upward from the sink area to show ceiling-mounted fluorescent light panels and ventilation grilles. The white walls and clean lines reinforce the sterile, functional aesthetic of the restroom. The camera then begins to turn back toward the toilet stalls, preparing to reframe the space from a different angle.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbdae9b4f96aa178086e4d3ced80b2fa4d8de0d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b18efefe823379fb0b5e45b9d83ff1d62e1aab9008a1282591e6d34959c273f1 +size 999331 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9165cc41f4e9d4b6b0d2538972e5f0ea99ce5802 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Returning to the toilet stalls, the camera offers a rear-facing view of the gray partitions, showing their alignment and spacing. The white porcelain toilets are visible through the gaps, and the speckled gray floor continues to dominate the lower frame. The camera slowly pans right, maintaining a steady pace to ensure continuity with the prior segments.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95fc67339d121924124c115f5ef61468b4eb2259 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d720f1c43bfd3f874b9f02a9282429a893e8eed763a12d0a3e99d1a5c8b5af4b +size 1696647 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..6dda98ae9d9aaac9a209afcdccca57230e5212c9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"The camera now sweeps across the entire restroom, combining views of both the sink counter and the toilet stalls. The mirror above the sink reflects the toilet stall once more, reinforcing the spatial relationship between the two areas. The blue bucket under the sink and the trash can with the white liner remain visible, anchoring the scene’s composition.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a2cb00443ead90a301defa4e1db5db3367e66da --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b3556c755421e009700e75ad6ecfbe028f5af8bb04c6dc108b77b43ed0389a7 +size 1144202 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..9b65eaf4178c50f0908ca72468917b070c034c6c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Finally, the camera pulls back for a wide-angle shot that encompasses the entire restroom. All mandatory landmarks are clearly visible: gray toilet stalls, white sink counter, wall-mounted mirrors, mirror above sink, reflection of soap dispenser, reflection of toilet stall, blue bucket under sink, and trash can with white liner. The video ends on this comprehensive, static overview, capturing the symmetry, cleanliness, and functional design of the space.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d30f5de70a36236dcc0de9c6fade861709c693a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bbc916354800a1403f670df83d3102b974a47b35ae172503dfa988454ccd467 +size 838224 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f373d75a3e79ed330a454725ae7258de56c7ded --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/console.log b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/console.log new file mode 100644 index 0000000000000000000000000000000000000000..3628d8185ba31c88e38601d63e08484371a2b999 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/console.log @@ -0,0 +1,22 @@ +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/config.yaml +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.json +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:29:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:29:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:29:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:29:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dbf9fef09910ba850d2dbb45d66db6ce7ed2846b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/debug.log @@ -0,0 +1,353 @@ +[02-20 07:15:45|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 07:15:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:15:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:15:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 07:15:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 07:15:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 07:15:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 07:15:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 07:15:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 07:15:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 07:15:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 07:15:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 07:15:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 07:15:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 07:15:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 07:15:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 07:15:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 07:15:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 07:15:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 07:15:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 07:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 07:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 07:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 07:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 07:15:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 07:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 07:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 07:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 07:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 07:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.67 s +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.68 s +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.68 s +[02-20 07:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:16:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 07:16:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 07:16:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.27 s +[02-20 07:16:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:16:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/config.yaml +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 07:16:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: Finally, the camera settles back into its starting position, offering a symmetrical composition of the first bench, the curving paved walkway, and the willow tree overhanging the lake. The green bucket under the distant tree, the wooden bench with black metal frame, the paved walkway, the willow branches, and the lake shoreline are all visible, creating a harmonious and complete visual loop that encapsulates the entire scene.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.json +[02-20 07:16:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:16:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.mp4 +[02-20 07:16:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 07:16:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 07:16:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 07:16:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 07:29:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 07:29:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:29:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:29:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:29:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 07:29:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..80c39573170ee07fbe541c894a105f178fabedac --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video begins with a medium shot of a wooden bench with a black metal frame positioned on a paved walkway that curves left along the lake shoreline. Overhanging willow tree branches frame the top of the frame, casting shadows on the grass below. The lake glimmers in the background, bordered by reeds and grasses. The camera is stationary but poised to begin a slow pan to the right.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8319f9592db398ecef631fb2839d262db837ba79 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a431dd6f6ad577bbc9452a8a5dd9522fc1f489f51d1f850b66ea944eb855986 +size 1797119 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..99399e7ed340db0f1c5fe5080f0979c7b765a829 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: The camera begins a smooth rightward pan, revealing more of the paved walkway and the continuation of the lake shoreline. The willow tree’s branches remain prominent, gently swaying. The first bench stays in view on the left as the camera uncovers a second bench further down the path, also near the water and under another willow tree. The green bucket under the second tree is not yet visible.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..698742a4aa7e8c70761de27333e7e3246d1adf45 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:883bfe7dfcdb02358c90eba8fd7851b8304005438997073d62cf95a4e89cbe79 +size 1450871 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..1109cbcd0d4d1c0827f49d3eeb8c4aecefd57375 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The pan continues, bringing the second bench fully into view. The green bucket under the tree becomes noticeable, resting near rocks and partially hidden by low-hanging branches. The paved path curves visibly to the left behind the bench, and the lake shoreline extends into the distance. The camera maintains a steady pace, emphasizing the tranquility of the setting.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..367774ad2cd116a36c7c099b014e643d80169957 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d1c5da592f78ec49094aa78dd22967701625f8cc3cd33adc6779bea95b90c0b +size 1270172 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..f5d37fe0721fd2cdbe89b91ce978ce979a0fb01c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The camera begins to circle around the second bench, shifting perspective to show the paved walkway extending behind it and the full expanse of the lake shoreline. Willow branches drape gracefully into the frame, enhancing the sense of enclosure and peace. The green bucket remains visible under the tree, adding a subtle human element to the natural scene.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9072b907b4a73eae257dd42f4b8df979e29d6a81 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a97952216689331b980ea51f6d9d2b346bc6394ca7ecc35c72e2900c734676a6 +size 1413620 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..97514f0605b6a5ac43ff3eb42aed91e6fbac0802 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: Continuing the circular motion, the camera now faces back toward the first bench, capturing the full length of the paved path that connects both benches. The willow tree overhanging the lake is prominent again, its branches framing the shot. The camera begins a slow forward tracking movement along the paved walkway, passing the first bench once more.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a87f575888a719cf683504930bac58d6dfa82067 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1659192ff703aba899192c993a02b63f2e0c4a3e86f29ac401843b1ac09ba2b +size 1523702 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..474a5431bd2440b9e6a9bd9c4dc63fdc9b991b64 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: As the camera tracks forward, the texture of the stone pavers and scattered dry leaves become more detailed. The grassy bank beside the path is uneven, with patches of dirt and moss. The willow tree branches overhead sway gently, filtering sunlight onto the path. The lake shoreline remains visible to the right, with reeds swaying slightly in the breeze.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ff7a850284258282d022086305448a335e5788d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c89067afc79692aec486a9bdb2d6a76db675611f11b09788951b0505a0c22e3c +size 1713168 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..b915e5e573424851ddcdd5d8e7360786613fd1d5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera continues its forward motion, approaching the end of the paved path near the first bench. It begins to arc back to the left, initiating a return to the original viewpoint. The second bench and the green bucket under the tree come back into view in the distance, framed by the willow branches and the lake beyond.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7a4c671e3295cbbbff4edf22d5a793cc0f48e88 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59d272f2cb3c0bf2e3a19ff0698ce9537e76d4a2ab135539c72f81818a8449d7 +size 1738658 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..96574a838317220494cdba012d2648937ce96c3c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: Finally, the camera settles back into its starting position, offering a symmetrical composition of the first bench, the curving paved walkway, and the willow tree overhanging the lake. The green bucket under the distant tree, the wooden bench with black metal frame, the paved walkway, the willow branches, and the lake shoreline are all visible, creating a harmonious and complete visual loop that encapsulates the entire scene.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..26b1c96ad62b955697074f968370be642be7de47 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f84e0877b27432e280a037c38f7a334c214a43fa56d97a1d13d21e5968bffb37 +size 1765383 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7baf259d0f5dd1be889a08ddb1f3b7db8083831e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/console.log b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/console.log new file mode 100644 index 0000000000000000000000000000000000000000..5a9a0424054f4501d828a736f4f86e2ce526e2e1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/console.log @@ -0,0 +1,22 @@ +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:52:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:52:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:52:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:52:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/config.yaml +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.json +[02-20 14:52:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:05:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:05:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:05:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:05:57|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..abb3bd4b3eefd9f75e3d47d244c8ab8f2baca8af --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/debug.log @@ -0,0 +1,353 @@ +[02-20 14:51:51|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 14:51:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:51:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:52:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 14:52:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 14:52:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 14:52:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 14:52:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 14:52:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 14:52:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 14:52:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 14:52:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 14:52:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 14:52:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 14:52:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 14:52:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 14:52:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 14:52:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 14:52:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 14:52:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 14:52:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 14:52:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 14:52:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 14:52:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 14:52:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 14:52:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 14:52:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 14:52:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 14:52:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 14:52:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:52:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:52:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:52:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 14:52:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 14:52:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:52:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 14:52:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 14:52:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:52:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:52:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 14:52:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 14:52:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.49 s +[02-20 14:52:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:52:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:52:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 14:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/config.yaml +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 14:52:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: The camera completes its full rotation, returning to a frontal view of the polka-dot office chair. The TV on the black stand with the cityscape display is centered behind it. The beige sectional sofa with the patterned blanket is on the left, and the other beige sofa is on the right. The floor lamp and framed pictures are visible in the background. The video ends on this stable, well-composed shot.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.json +[02-20 14:52:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.mp4 +[02-20 14:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 14:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 14:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 14:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 15:05:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 15:05:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:05:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:05:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:05:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 15:05:57|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..5dc1e6815f6bccb583f93de39eb62aa454e37439 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a low-angle, wide shot focused on a black office chair covered in colorful polka-dot fabric (purple, orange, beige circles). The chair has a black five-star base with casters. To the left, a beige sectional sofa with a patterned blanket is visible. The camera begins a slow clockwise rotation. The carpeted floor and ambient lighting are clear.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3dd0ba916b9c60baf1ae2392f108e95e717d0176 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cab1d0368e22052a2e13d84ec1889e6a06afbf7e5d2ce2282c5498e79ecf5b0 +size 1418129 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..3032eb30b0757a79283fb757700ed22edf5e216e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the rotation, the right side of the room comes into view — a beige sofa with matching polka-dot pillows. Behind it, a floor lamp and wall-mounted framed pictures are visible. The camera maintains focus on the polka-dot chair while revealing more of the living room layout, including a TV on a black stand in the background.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5d6341edaf0c2a6a4fb0bdddd01dbe80f67b0a9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3be1a6f677e085fa14dbcd9ee032f21e134640fe484e1acdd26ad04338e2ce17 +size 1113200 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..ad34d59055a8fe9dd709b1d85684ac824614811c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera continues rotating clockwise, bringing the TV into clearer view. It displays a bright cityscape with skyscrapers. The polka-dot chair remains centered. On the far left, a brown leather office chair near a desk becomes briefly visible. The overall lighting remains consistent and warm.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..645abf9879cfa888fc9d05d79bf26d5240474c7f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77a13c4a81399284991b4dce109b034d6ba0cb7f27ae0b093818d1df39c67b98 +size 1531948 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..492ab79b8412e962908ec38a9527c1af7c3f84ee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: The rotation progresses further, showing the brown leather office chair more clearly next to a small desk. The camera angle begins to lower slightly, preparing to tilt down. The beige sofas and patterned blanket remain visible on either side, framing the central polka-dot chair.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e4fbeaaf17852840c720f1b9be086a4be4b1ae50 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abeac093727b2cb427efb1d80a916c0394c3381ac37caff9e364214085da40a6 +size 1530948 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..776799e52acfe5e797653b87a3a4c9dadc422aa3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera tilts sharply downward, moving beneath the polka-dot chair. The blue pneumatic lift cylinder and the black plastic base mechanism become the focal point. The five-star base with casters is clearly shown rolling on the light-colored carpet. The underside of the seat and mesh backrest are briefly visible.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1d352f1d7ba5a0d74de7859d215f77ff69a6dda3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b57a0d0efecd125dbf3ad7f32b1bc8a8eb9abbd4077b27d40d54ea8800766260 +size 1052739 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..8f89f8ad7a082346ab9bfa84013ead0e2b1a0b98 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera pans upward slightly while still positioned beneath the chair, highlighting the mesh texture of the backrest and the contours of the armrests. The blue pneumatic cylinder remains prominent. In the blurred background, glimpses of the beige sofas and the TV screen can be seen.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bc5f91379d442185a0475fc95a22e019d515812b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c89d474ddfb88a355dc13ee1cae58c074bb5d5c2e8811938549ed6ce230fb7aa +size 1200251 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..797e3d79dda3e6476a3abbc02f490419371fa0e3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: The camera begins to pull back and tilt upward, rising from under the chair. The polka-dot fabric and black frame come back into full view. The camera resumes its slow clockwise rotation, bringing the TV displaying the cityscape and the wall-mounted framed pictures back into the frame.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..406b7357b471b0de7c93b5d57fca0f5b6fdb9488 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4675dd234f4ce5001ec4344966eb45942b73b7ad406079fb01e065a57091b8f2 +size 1105671 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..4f3db8973d89a12a14aca89d7b2ba6b4852c1e26 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: The camera completes its full rotation, returning to a frontal view of the polka-dot office chair. The TV on the black stand with the cityscape display is centered behind it. The beige sectional sofa with the patterned blanket is on the left, and the other beige sofa is on the right. The floor lamp and framed pictures are visible in the background. The video ends on this stable, well-composed shot.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9bb52ffb3bd48f6cedae376838ec8b577ecec08b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b755fa7a8aaeae84ce91968286c2859b048326a09ca1963348bf33d0c7b1185 +size 1329503 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/config.yaml b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f636460b43f188c8963facc5d85216a8fa272281 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/console.log b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/console.log new file mode 100644 index 0000000000000000000000000000000000000000..75c4d6b1739ebcee9c184b8817116cbc25081092 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/console.log @@ -0,0 +1,22 @@ +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:05:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:05:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/config.yaml +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.json +[02-20 01:05:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:18:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:18:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/debug.log b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a7b2d1d20ef935c0163a6b78a9a937d82669655a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/debug.log @@ -0,0 +1,353 @@ +[02-20 01:04:45|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 01:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 01:04:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 01:04:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 01:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 01:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 01:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 01:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 01:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 01:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 01:04:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 01:04:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 01:04:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 01:04:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 01:04:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 01:04:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 01:04:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 01:04:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 01:04:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 01:04:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 01:04:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 01:04:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 01:04:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 01:04:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 01:04:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 01:04:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 01:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 01:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 01:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 01:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 01:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 01:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 01:04:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 01:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 01:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 01:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:05:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:05:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 01:05:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:05:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 01:05:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:05:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 01:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 0.92 s +[02-20 01:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 01:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 01:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 01:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 01:05:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 01:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 01:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:05:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 01:05:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 01:05:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.46 s +[02-20 01:05:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:05:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:05:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/config.yaml +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_7'] +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_7 +[02-20 01:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_7' prompt_path=None prompt='Part 8/8: The camera completes its 360-degree orbit, returning to the initial framing. The central statue on pedestal is centered once again, surrounded by the curved pathway, green lawn, and trees. The blue sky with clouds remains overhead, and both the modern glass skyscraper and the neoclassical building with the dome are visible in their original positions. The video ends with a lingering shot that encapsulates the harmony between nature, history, and modernity.' negative_prompt='' seed=49 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.json +[02-20 01:05:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.mp4 +[02-20 01:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 01:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 01:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 01:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 01:18:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 01:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:18:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:18:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 01:18:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.mp4 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_0.json b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..127f2661024e3405b1ac8122a0663a29eb8b7753 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"Part 1/8: The video opens with a wide-angle shot of a grand stone monument featuring a central statue on a pedestal, set within a verdant park. Green lawn and tall trees surround the structure under a clear blue sky with wispy clouds. A curved pathway encircles the monument. In the background, a neoclassical building with a dome is visible, and to the left, a modern glass skyscraper reflects the sunlight. The camera begins a slow clockwise pan around the monument.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_0.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3c7e11f0334a9f31891758342c5aff04efd383fb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69a9be4789243731fa12e2cec5024ee38f8e7ce07d53b6a0a61c507d79b57f04 +size 1202970 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_1.json b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..328f961ba40678e0f11dd4e8b4ff839f1272517f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Part 2/8: Continuing the orbit, the camera moves slightly left, bringing the modern glass skyscraper into greater prominence while the neoclassical building remains partially visible behind the monument. The central statue looms large against the blue sky. Details of the stone base with sculpted figures — including a seated draped figure — become more discernible. Sunlight creates lens flares as the camera tilts slightly upward, highlighting the monument’s verticality.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_1.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b9baa133f45ba078c9dc774574b9bba0dae06f6f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05e10aac0d99fa56da82a4c9ba7b2b60d7751f88e79a69a0e22ec1de15847d4e +size 1801568 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_2.json b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a604c19798cec05c4ca46201fea0e6c3a9294bd8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Part 3/8: The camera continues its circular motion, now positioned to show the monument’s left flank. The curved pathway is clearly visible winding around the base. Trees frame the edges of the shot, and the green lawn extends toward the foreground. The modern glass skyscraper dominates the left background, while the neoclassical building with the dome peeks from behind the trees on the right. The sculpted figures on the stone base are shown in profile.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_2.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4812d23c6aa619198957aca51d8b5a9eb57909ba --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0faf25f37f14b0e286e64046b5ccf1f5997d3e76390bd7ce94c7aad59eefb14b +size 1484255 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_3.json b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9faebdb04f9e8855a3214a370b9cff9034dcf641 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Part 4/8: Midway through the orbit, the camera focuses on the rear of the monument. The central statue appears silhouetted against the bright blue sky. The stone base’s sculpted figures — including a gesturing allegorical figure — are now seen from behind. The curved pathway continues to curve around the structure. The neoclassical building with the dome is temporarily obscured by trees, while the modern glass skyscraper remains visible to the far left.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_3.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a1d7ac49970981d1a437a6fd29e41213777d994 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3afe28fb2004224ea5ddc7fe0eaafdeef43513ace20afedca6bdee11099a4a8 +size 1373983 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_4.json b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e36e3fdb8f14881c99a2ca0f5574fe92cfea403c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Part 5/8: The camera moves to the right side of the monument, beginning to reveal the front again. The green lawn and trees remain consistent in the foreground and background. The sun casts long shadows, enhancing the texture of the stone base and sculpted figures. The neoclassical building with the dome re-emerges in the background, now more centered. The modern glass skyscraper is still visible on the far right edge of the frame.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_4.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ff21cc9181c9ea8c2d1abc840cd50994ae603dd8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ee96c6cf0670857377fbf979852256f6a6de34ea9d3fe4f287486fb03ce137f +size 1381088 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_5.json b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_5.json new file mode 100644 index 0000000000000000000000000000000000000000..3bb73ddbf46e94b5a0fac550831ac41f303bcca9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_5.json @@ -0,0 +1 @@ +{"name":"segment_5","prompt_path":null,"prompt":"Part 6/8: The camera continues its clockwise arc, now showing the monument’s right flank. The central statue on pedestal is framed between the neoclassical building with the dome and the modern glass skyscraper. The curved pathway curves gracefully around the base, leading the eye through the green lawn. The sculpted figures on the stone base are illuminated by direct sunlight, revealing fine details in their drapery and posture.","negative_prompt":"","seed":47,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_4.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_5.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67b38cb117626b1c135daa1e72e01f0dd72ae450 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b9188bf206d2f7edf8dbb93606b3bde6a3e2a1d6236a4a60549e0ccbedead25 +size 1427694 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.json b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.json new file mode 100644 index 0000000000000000000000000000000000000000..60a9a8f59619f07d098583d70e33618c15a4bb5e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.json @@ -0,0 +1 @@ +{"name":"segment_6","prompt_path":null,"prompt":"Part 7/8: As the camera approaches the final quarter of its orbit, it tilts upward briefly to emphasize the central statue against the blue sky with clouds. Then it lowers to capture the full stone base with sculpted figures, now viewed from a three-quarter angle. The green lawn and trees frame the scene symmetrically. The neoclassical building with the dome is fully visible behind the monument, and the modern glass skyscraper appears on the left, completing the architectural contrast.","negative_prompt":"","seed":48,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_5.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68e157b566fdd066175ed8904c33650b00cf774b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:872c5184210424947d5547c5ade0b48831274c83659b1a6b7302372f45af1a88 +size 1368914 diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.json b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45888622fd125e69e1d490a0634d1c530db58e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.json @@ -0,0 +1 @@ +{"name":"segment_7","prompt_path":null,"prompt":"Part 8/8: The camera completes its 360-degree orbit, returning to the initial framing. The central statue on pedestal is centered once again, surrounded by the curved pathway, green lawn, and trees. The blue sky with clouds remains overhead, and both the modern glass skyscraper and the neoclassical building with the dome are visible in their original positions. The video ends with a lingering shot that encapsulates the harmony between nature, history, and modernity.","negative_prompt":"","seed":49,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_6.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.mp4 b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb726642e61f0e9f8a8aa365c134b5eb064d3c76 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580/segment_7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc837b4249eed1a167ec37fdc15116e3cae932eccbff52aaad20aa5d663bfc5d +size 1583786 diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f1285755f63e6707d29758a4ae9ba401c71e05a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/console.log b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/console.log new file mode 100644 index 0000000000000000000000000000000000000000..8e889294a3d91c7c1988f9dbc1ce1836e83ea19e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/console.log @@ -0,0 +1,22 @@ +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:16:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:16:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/config.yaml +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.json +[02-20 07:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:30:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:30:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:30:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:30:07|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/debug.log b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b5192edbb32d52dc200705594f591b9283346200 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/debug.log @@ -0,0 +1,353 @@ +[02-20 07:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 07:16:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:16:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 07:16:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 07:16:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 07:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 07:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 07:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 07:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 07:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 07:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 07:16:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 07:16:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 07:16:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 07:16:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 07:16:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 07:16:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 07:16:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 07:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 07:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 07:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 07:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 07:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 07:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 07:16:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 07:16:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 07:16:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 07:16:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 07:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:16:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:16:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:16:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 07:16:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:16:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 07:16:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:16:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 07:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 07:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 07:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 07:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 07:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:16:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 07:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:16:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 07:16:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 07:16:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.43 s +[02-20 07:16:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:16:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:16:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/config.yaml +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 07:16:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='The video concludes with the mechanic making final adjustments to the white plastic front fairing panel, ensuring it is perfectly aligned and securely fastened. The mechanic steps back, inspecting the work to confirm that the panel is properly installed. The video ends with the mechanic standing beside the motorcycle, having completed the repair task.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.json +[02-20 07:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.mp4 +[02-20 07:16:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 07:16:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 07:16:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 07:16:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 07:30:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 07:30:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:30:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:30:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:30:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 07:30:07|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f75cd7eecee99ce2545dd4f20131ee378bebe170 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a mechanic in a workshop, wearing a blue shirt, working on a motorcycle. The mechanic focuses on the front section of the motorcycle, specifically the white plastic front fairing panel, which is attached to the frame with multiple mounting points. The mechanic begins by using a tool to loosen and remove the screws securing the panel.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bf48d438605446fc3627035bfb2baa8ea55f0cf2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0b32e965d115a5dbe2f307f9c8706053a8955c9cf4445a9192d77ddcfad3161 +size 742699 diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..d84ec35593bf63f97d85667e98a3c2facc4f7cce --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the mechanic continues to remove the screws from the fairing panel. The panel is carefully detached from the motorcycle frame and placed on the workbench. The mechanic then retrieves another tool from the workbench, preparing to proceed with the next step in the repair process.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..757557a3e6443a764a0a2d0c6df9bd0c003cacd2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:737f78d76a94856956778f5b1e663314319979676ef9ef34e83203d1c80dbb68 +size 1378073 diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..b3e7338143bc6913708b9772882772668d406644 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The mechanic then reinserts the white plastic front fairing panel into its original position on the motorcycle. The panel is aligned with the frame, and the mechanic begins to tighten the screws using the tool. The mechanic ensures that each screw is securely fastened, paying close attention to the alignment and fit of the panel.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d48c53d5ec52f00cbb3345ea73f65fbb35c8d442 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cc4c37659924f84b274a96d58ce7f8323b2a180caa9d84cde0b273827933d66 +size 1436215 diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9b794c2667e1aa8dd84a18a58d308d66dcf29bb2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the following segment, the mechanic continues to tighten the screws on the fairing panel, making sure it is firmly attached to the motorcycle frame. The mechanic checks the alignment of the panel, making minor adjustments as needed. The workbench in the background remains cluttered with various tools and parts.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1f54ae40a907fb0a1229598f166249156b075c29 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e2c126019f6a34af3df773bfdd722f4efe26a78f81c0bd4651482cf4374bf59 +size 1289685 diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..47daa69eb16b3dfc65d540bb159e32a1323d1d47 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with the mechanic making final adjustments to the white plastic front fairing panel, ensuring it is perfectly aligned and securely fastened. The mechanic steps back, inspecting the work to confirm that the panel is properly installed. The video ends with the mechanic standing beside the motorcycle, having completed the repair task.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6f403911e551a4b0529a33230d1825230b21998a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:667b8008535dca49ad72eb58b9cf01c5555a670c48dd77fba216f1db9d3c750c +size 1342502 diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c532eeaf716882818420bad20902c3a5a69839bb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/console.log b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/console.log new file mode 100644 index 0000000000000000000000000000000000000000..7ddcee805a800353b6f1d6f51b22860883fde66a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/console.log @@ -0,0 +1,22 @@ +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/config.yaml +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.json +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:41:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:41:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/debug.log b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e725a300c7a2c6e6ca63d413ac85991fdd50eb23 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/debug.log @@ -0,0 +1,353 @@ +[02-20 08:27:12|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 08:27:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:27:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 08:27:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 08:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 08:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 08:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 08:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 08:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 08:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 08:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 08:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 08:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 08:27:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 08:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 08:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 08:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 08:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 08:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 08:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 08:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 08:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 08:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 08:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 08:27:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 08:27:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 08:27:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 08:27:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 08:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 08:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:27:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 08:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 08:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 08:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 08:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 08:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 08:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 08:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 08:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 08:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.84 s +[02-20 08:27:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:28:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:28:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/config.yaml +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:28:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the person closes the cabinet door and walks back to the sink area. They pick up a blue sponge and begin wiping down the sink basin and surrounding countertop, ensuring everything is clean and tidy. The video ends with the person finishing up the cleanup, leaving the sink area neat and ready for the next use.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.json +[02-20 08:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.mp4 +[02-20 08:28:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 08:28:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 08:28:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 08:28:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 08:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 08:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:41:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:41:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 08:41:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..cfae69dc4a23d7e11c3a9538c3b3a713dc92c8a7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a first-person view of a person washing a black frying pan with a red interior in a stainless steel kitchen sink. The pan has a black handle and is metallic. The person uses a blue sponge to scrub the inside and outside of the pan while water flows from the faucet. The countertop around the sink is cluttered with a blue dish rack, green containers holding utensils, and various cleaning products like dish soap and sponges. The backsplash is made of blue tiles.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64509e3f66d1fac89d39b91a2c93d1556bfbe5dd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40402e689624e25ea7c253173885d2ee70ef1c413f79bf6a2336ab8cdb8c4f04 +size 1512280 diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2d017bf1c5443ada5e36868d8acfe4b98b63f3b6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"After scrubbing the pan, the person continues to rinse it thoroughly under the running water, ensuring all soap residue is washed away. Once clean, they lift the pan from the sink and place it on the stovetop nearby. The camera follows their movement as they walk toward a white cabinet located near the microwave, open the door, and retrieve a yellow plate from inside.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4617bac9d9ea25e2724c42508429f2e09d333adb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d66844153f1c436826eba9ffb41db2dff0db02f00731c98f420ec6a888dfa4d +size 2240844 diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..699bb71f7094a333acd64526afaec2cc4892880a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person places the yellow plate on the countertop next to the sink. They then return to the sink area and use a blue cloth to wipe down the countertop, cleaning any remaining water or soap. The camera captures the organized yet busy kitchen environment, including the blue dish rack, green containers, and various items on the counter.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..757aeb96a44ca3deb8db10af65c992c90586ef48 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b29b077ed4f4bb97ff027b8ffe062e99525cbc619d9c712ff92b9c857ad005b +size 1514629 diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..06b4528ad80d2a51d938ffc374d008670c6ca176 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Next, the person moves to a different section of the kitchen, opening a lower cabinet beneath the microwave. Inside, they retrieve a clear glass lid and place it on top of a yellow bowl already on the counter. The camera pans slightly to show more of the kitchen, including a refrigerator and additional storage areas.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d9db2133b470dbd101569cabab753a41071aeba7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abbf84cbe5b230cba762f57a52948ddc2f885582ec5d2f75452f5eaa5aea90a6 +size 1851281 diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..f4e038780c786345eeebfe43235455e61f0c8b09 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person closes the cabinet door and walks back to the sink area. They pick up a blue sponge and begin wiping down the sink basin and surrounding countertop, ensuring everything is clean and tidy. The video ends with the person finishing up the cleanup, leaving the sink area neat and ready for the next use.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d677540e5c1159da1a7bb813f12b4862672f4ce2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac2717b837fdc28fe513c657532934587cb1165f179162ccddee9535a4dfac4a +size 1401188 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e1d3a0963be71c1e877b040e8e09e71b981fd64 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/console.log b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/console.log new file mode 100644 index 0000000000000000000000000000000000000000..44ddd04a484bef534e369aad139f72147ec3884a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/console.log @@ -0,0 +1,22 @@ +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:32:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:32:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/config.yaml +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.json +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:46:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:46:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:46:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:46:17|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/debug.log b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e2cb36c2dc9fb3f5864915c73454748a16f2bb4e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/debug.log @@ -0,0 +1,353 @@ +[02-20 02:32:06|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 02:32:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:32:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 02:32:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 02:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 02:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 02:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 02:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 02:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 02:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 02:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 02:32:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 02:32:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 02:32:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 02:32:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 02:32:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 02:32:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 02:32:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 02:32:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 02:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 02:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 02:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 02:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 02:32:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 02:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 02:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 02:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:32:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:32:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:32:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 02:32:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:32:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 02:32:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:32:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:32:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 02:32:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:32:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 02:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 02:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.04 s +[02-20 02:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:32:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:32:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/config.yaml +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 02:32:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final segment, the manicurist makes final adjustments to the client's nails, ensuring a flawless finish. The camera provides a close-up view, capturing the detailed work being done on each nail. The client's hand, still adorned with vibrant orange polish, is held securely as the manicurist uses a small brush to clean any remaining excess polish. The video concludes with the manicurist making sure each nail is perfectly shaped and polished, with the client's hand resting on the white towel." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.json +[02-20 02:32:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:32:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.mp4 +[02-20 02:32:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 02:32:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 02:32:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 02:32:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 02:46:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 02:46:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:46:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:46:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:46:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 02:46:17|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..36feb9caa3e89b4e44d4c5c52868929ea32bc8f5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a top-down view of the client's left hand, showcasing long, almond-shaped nails painted in bright orange. The manicurist, wearing a red bracelet, begins by cleaning the nails with a small tool, holding the hand steady with the other hand. The client's hand rests on a white towel, and the background includes a blue basket with various nail care products. The camera angle is slightly adjusted to provide a clearer view of the manicurist's precise movements.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c97724b310f2828d8e153e0858eab0a82bda8a4e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7f016bcb0e8d68d0917e68e7f6d971c367fa2f304a4e768198c0e2734e00d71 +size 832529 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..23dfe954f23fa71aa6a9b96270ef2e3197685ac5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the manicurist focuses on shaping the nails using a blue-handled device, likely a buffer or file. The camera shifts to a closer perspective, highlighting the detailed work being done on each nail. The client's hand remains steady on the white towel, and the background shows the blue basket with nail care products. The manicurist moves methodically from one finger to the next, ensuring each nail is perfectly shaped.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4c15755ea52a41363ad019ee4a2533929a6f36b1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4923255f97081d23752f6b0e8336fb0d6f38b767ae0ab4cc0ed08ab00ef5013b +size 944714 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4fc58d8e12a9ded9396ccc939b74ffb0a54e3f1f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The manicurist then uses a small brush to clean the edges of the nails, removing any excess polish. The camera angle changes to provide a side view, capturing the careful and precise movements of the manicurist's hands. The client's hand, still with vibrant orange polish, is held securely as the manicurist ensures each nail is clean and polished. The background remains consistent with the blue basket and tiled floor visible.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a287ee608bdae26c2b7d1f614f5d3ec4413ad561 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab0ce6f88a542203e40972ecadec96357f8fbf9bc92df9f3c8b647b886caa30d +size 1015756 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..83c90513cb944788b7adfe85fb7448f297dcf90c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Continuing from the previous scene, the manicurist refines the nails further, focusing on the tips and edges. The camera angle shifts slightly, offering a top-down view that emphasizes the meticulous work being done. The client's hand remains on the white towel, and the background includes the blue basket with nail care products. The manicurist uses a combination of tools to perfect the shape and polish of each nail.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c82c9c3b93f328188b181bb59026111c15380f93 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9f265e798b1627d93df504f5da0f36cc43a1153a8cba33465f10fc8d2217fad +size 1140988 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..1060aabcfeae362cc3555e17cd68a59af52b3070 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the manicurist makes final adjustments to the client's nails, ensuring a flawless finish. The camera provides a close-up view, capturing the detailed work being done on each nail. The client's hand, still adorned with vibrant orange polish, is held securely as the manicurist uses a small brush to clean any remaining excess polish. The video concludes with the manicurist making sure each nail is perfectly shaped and polished, with the client's hand resting on the white towel.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a53d95d50e3d1d0b230f10dab1b1a8479bf44ecf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:111fc042bd5bbaa5f1be6f6421f595ec6e6a41f38572244e7f07c637ef438cc7 +size 946489 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c46ed6049188b35ce9b6bbe20b17da72248c3e8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/console.log b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/console.log new file mode 100644 index 0000000000000000000000000000000000000000..5ab22291a42adb0c8477b3f4415669af9be9d31f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/console.log @@ -0,0 +1,22 @@ +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:04:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:04:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/config.yaml +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.json +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:18:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 11:18:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 11:18:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 11:18:44|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/debug.log b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..337adbcda0457111737082519af20ca91f5a118a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/debug.log @@ -0,0 +1,353 @@ +[02-20 11:04:38|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 11:04:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:04:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:04:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 11:04:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 11:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 11:04:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 11:04:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 11:04:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 11:04:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 11:04:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 11:04:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 11:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 11:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 11:04:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 11:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 11:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 11:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 11:04:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 11:04:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 11:04:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 11:04:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 11:04:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 11:04:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 11:04:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 11:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 11:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 11:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 11:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 11:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 11:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 11:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 11:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 11:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 11:04:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 11:04:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 11:04:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:04:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:04:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 11:04:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:04:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:04:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 11:04:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:04:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 11:04:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:04:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 11:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 11:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 11:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 11:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 11:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 11:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:04:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:05:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 11:05:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 11:05:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.86 s +[02-20 11:05:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:05:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:05:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/config.yaml +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 11:05:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the person cleans up the workspace by washing their hands under running water in the stainless steel sink. They dry their hands with a blue cloth and tidy up the counter, completing the task.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.json +[02-20 11:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.mp4 +[02-20 11:05:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 11:05:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 11:05:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 11:05:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 11:18:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 11:18:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 11:18:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 11:18:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 11:18:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 11:18:44|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..eeb730a41b093a08f8170b077c022e721bcd2eff --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a person in a blue sweater opening a freezer drawer in a kitchen with tiled walls. Inside the drawer, there are various vegetables, including a carrot and a sweet potato. The person takes out the carrot and places it on the white countertop next to an onion.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb39c8da0b2d352c77bffb3f9bb7079be80536a1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b699cc2d1cca1d3cb8b7e333c0031682bd01978341700464d77e6008d1f40ea2 +size 1103706 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..4f14f671a9d95d6396033a2f18d9238724896d0b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Next, the person moves to a nearby drawer and selects a metal vegetable peeler. They return to the countertop, holding the carrot in one hand and the peeler in the other, preparing to begin the peeling process.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6dbd87fe68db94e07f8559e847f860772225438a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:672e34334ff75c15d92b4ce2f93d5281f2e4dd6637be1ad40c2f8ad8bf9b1ece +size 1294046 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..61a4892488ef8fa39041d2370313f85617f1f62c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person starts peeling the carrot, holding it firmly in their right hand and using the peeler in their left hand. As they peel, long strips of orange skin fall onto the counter. The onion and sweet potato remain on the counter as the person focuses on the carrot.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6b510f87b6a588531a0372e8d033e134d7be0f8d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d56a823e174152b7103ce7da26eeab7bbd3afa8dfaa58d91b066c9687b54bcf3 +size 1305408 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..984aa7b97d476dc39bdba9c9f7c2c67aee396567 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Continuing the peeling, the person ensures the entire surface of the carrot is free of skin. The pile of peels grows larger on the counter. Once the carrot is fully peeled, they set it down and gather the peels into a plastic bag.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67a68a4b84ae4e39a120b66956fefc61cbce744b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c09ad6a96c91a5ec5f4d4237684a41921ef0553571a88181e63fe3435d8d9e2e +size 1293436 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..9684c4471ed97b66135f7dc2351093dc0a805a0d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person cleans up the workspace by washing their hands under running water in the stainless steel sink. They dry their hands with a blue cloth and tidy up the counter, completing the task.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e5b41398fe486a305919f616e6cf46e54c1a0eb8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1868c602dccfbb6a804f20e8b8e12350e6d455e639e2c59b1a22a50c139543c +size 1455495 diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a303efdf7d176f773ec0206993bd5ddf64c1b51 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/console.log b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/console.log new file mode 100644 index 0000000000000000000000000000000000000000..1c8b65c14cb40eeda95d4ecccf16fd4b8fa870cb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/console.log @@ -0,0 +1,22 @@ +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/config.yaml +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.json +[02-20 08:35:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:48:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:48:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:48:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:48:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/debug.log b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..44e1a9e4e456d1454a0c4b518c3ed3bcf6038e40 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/debug.log @@ -0,0 +1,353 @@ +[02-20 08:34:36|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 08:34:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:34:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:34:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 08:34:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 08:34:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 08:34:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 08:34:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 08:34:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 08:34:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 08:34:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 08:34:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 08:34:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 08:34:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 08:34:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 08:34:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 08:34:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 08:34:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 08:34:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 08:34:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 08:34:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 08:34:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 08:34:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 08:34:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 08:34:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 08:34:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 08:34:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 08:34:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 08:34:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 08:34:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 08:34:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 08:34:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 08:34:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 08:34:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 08:34:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 08:34:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:34:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:34:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 08:34:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:34:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 08:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:34:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 08:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:34:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:34:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:35:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 08:35:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 08:35:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.84 s +[02-20 08:35:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:35:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:35:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 08:35:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/config.yaml +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:35:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the person continues to handle the metallic tool in the black container, making sure it is properly positioned and submerged. The workspace is still visible, with the same array of tools and supplies scattered around, indicating that the task is part of a larger, detailed process. The video concludes with the person still working on the tool, suggesting that the maintenance or preparation is not yet complete.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.json +[02-20 08:35:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:35:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.mp4 +[02-20 08:35:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 08:35:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 08:35:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 08:35:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 08:48:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 08:48:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:48:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:48:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:48:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 08:48:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e027818333fda017592feefcfb8f09daad8ed62a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up view of a person's hands, wearing black gloves and revealing a tattooed arm, as they hold a small metallic tool on a white paper-covered workbench. The tool, which appears to be a specialized instrument with a thin, elongated shape, is being manipulated with precision tweezers. The workspace is cluttered with various items, including a spray gun, a yellow cup, and a metal bowl, suggesting a detailed and technical task.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88cfb96ec0336e9e253a28290b608395693383fc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e442cf71ce3e96758c9ef549a0e3471d7e8ba632405d15e30d9f5ea84c2ec2d +size 808544 diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2d08d999e51638fb21cb8fc34754fee978cbe47a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"In the next segment, the person sets the metallic tool down on the white paper and reaches for a red toolbox drawer located beneath the workbench. They open the drawer and begin searching through the organized tools inside, eventually retrieving the metallic tool again. The workspace remains in view, with the same array of tools and supplies scattered around, emphasizing the meticulous nature of the task.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8950182291e680e1ebb449a29a98a7b52c67ab90 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a999d9e470ed7db4cada3d9bf9f17f550d453fc5a6df79b00f5ed173b7a8621 +size 1392840 diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..52829a56de5b046db4e5651d24a3deaba86da929 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The third segment shows the person resuming their work on the metallic tool, using the tweezers to make precise adjustments. The tool is carefully positioned on the white paper, and the person ensures it is properly aligned. The workspace is still visible, with the spray gun, yellow cup, and metal bowl in the background, reinforcing the setting of a detailed and technical environment.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9d07ffcbe3342733359bfada412eb051e4683679 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b65ab0d9ffad606c68f1d912971c4d4941afba48b0b79763b4d9c6d9b89d90e +size 856358 diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..f6b797d70f23e3ef53916cfbaacf3f788f07d750 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Moving forward, the person places the metallic tool into a black container filled with liquid, likely for cleaning or lubrication. They use the tweezers to submerge the tool completely, ensuring it is fully immersed. The workspace remains cluttered with various tools and supplies, including a spray gun, a yellow cup, and a metal bowl, highlighting the ongoing maintenance or preparation process.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a34f64568aa7f2445a52faca5497cd1273ed5d6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:487d9d111ce2390d11d5e8f29df76422e13359f3b767d6aba50bfbbef2490b3c +size 780648 diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..45a81eb55c3d4aa9db5f256d996c18a445c1371b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the person continues to handle the metallic tool in the black container, making sure it is properly positioned and submerged. The workspace is still visible, with the same array of tools and supplies scattered around, indicating that the task is part of a larger, detailed process. The video concludes with the person still working on the tool, suggesting that the maintenance or preparation is not yet complete.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85045b55751364fa522d0987826fe3fcdfe6f5ee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8ea12cd60aa2ffd812142e34e12ed0611259032b8b1e326c8c0ce7d236aa6ce +size 931782 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f2e9f5a13b669fd6bbc11f1e88205c7f16e97ae --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/console.log b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c1271dd3b7f3f139a5c92e71da771a75aa3c9048 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/console.log @@ -0,0 +1,22 @@ +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/config.yaml +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.json +[02-19 21:02:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:15:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:15:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:15:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:15:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/debug.log b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f1e520e918ffa56107916848d2ecaaa5fb3df1a7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/debug.log @@ -0,0 +1,352 @@ +[02-19 21:01:22|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 21:01:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:01:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 21:01:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 21:01:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 21:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 21:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 21:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 21:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 21:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 21:01:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 21:01:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 21:01:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 21:01:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 21:01:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 21:01:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 21:01:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 21:01:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 21:01:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 21:01:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 21:01:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 21:01:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 21:01:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 21:01:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 21:01:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 21:01:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 21:01:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 21:01:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 21:01:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 21:01:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 21:01:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 21:01:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 21:01:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 21:01:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 21:01:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 21:01:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:01:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:01:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 21:01:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 21:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-19 21:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 21:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 21:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 21:01:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:01:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.15 s +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.71 s +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.72 s +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.72 s +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:01:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:01:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:02:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 21:02:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 21:02:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.65 s +[02-19 21:02:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:02:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:02:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 21:02:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/config.yaml +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 21:02:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='After ironing the third shirt, the person folds it neatly and places it aside with the other two shirts. The video concludes with the person stepping back, having completed the ironing task for all three white t-shirts, each with its unique text and logo designs.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.json +[02-19 21:02:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 21:02:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 21:02:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 21:02:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 21:15:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 21:15:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:15:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:15:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:15:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 21:15:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..b26e86c72fe641ff46bdd7aaaae0f40591baa876 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"After ironing the third shirt, the person folds it neatly and places it aside with the other two shirts. The video concludes with the person stepping back, having completed the ironing task for all three white t-shirts, each with its unique text and logo designs.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..574669ceb23a2ef70ed4052db6a86ecfa65f4f2b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c390f50be443e4cc8e62eb1265dcc6e93265c3ef803830ac1243d12b8b34686 +size 855580 diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f81f71c934e0151e30e36c6522fa2f216d4ff63 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/console.log b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..26df2d8b2668e2a4f993a106ac379c721bdcd14f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/console.log @@ -0,0 +1,22 @@ +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 05:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 05:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/config.yaml +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.json +[02-20 05:07:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 05:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:20:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:20:43|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/debug.log b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7ca34efc9ca992c01462a78ea712e5b680edb2de --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/debug.log @@ -0,0 +1,353 @@ +[02-20 05:06:37|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 05:06:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 05:06:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 05:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 05:06:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 05:06:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 05:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 05:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 05:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 05:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 05:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 05:06:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 05:06:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 05:06:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 05:06:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 05:06:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 05:06:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 05:06:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 05:06:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 05:06:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 05:06:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 05:06:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 05:06:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 05:06:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 05:06:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 05:06:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 05:06:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 05:06:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 05:06:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 05:06:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 05:06:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 05:06:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 05:06:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 05:06:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 05:06:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 05:06:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 05:06:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 05:06:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 05:06:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 05:06:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 05:06:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 05:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 05:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:06:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 05:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 05:06:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 05:06:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 05:06:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 05:06:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 05:06:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 05:06:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 05:06:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 05:06:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:06:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 05:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 05:06:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:07:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 05:07:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 05:07:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.92 s +[02-20 05:07:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:07:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:07:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 05:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/config.yaml +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 05:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='The video concludes with the person nearing the end of the hallway, continuing to apply plaster to the ceiling. The camera captures their movements as they reach higher sections of the ceiling, ensuring even coverage. The fisheye lens provides a distorted view of the surroundings, including a white door and a window. The person occasionally steps down to reload the putty knife with more plaster from the container.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.json +[02-20 05:07:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 05:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.mp4 +[02-20 05:07:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 05:07:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 05:07:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 05:07:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 05:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 05:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:20:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:20:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 05:20:43|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..6b236cd2d30033520b78fea8483c487713c955df --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with the person holding a metal putty knife with a wooden handle and a rectangular metal container filled with plaster. They are standing in a narrow hallway with unfinished walls, some of which have patches of blue and yellow paint. The fisheye lens creates a distorted view, making the putty knife appear larger and closer. The person scoops plaster from the container and begins applying it to the textured ceiling, moving along the hallway.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..167ba3d0a1c82dbc89002b7a5b043790e01d71ab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd119523f3a09766d18739fc575d74f410998e7e69eb500d945a1f2e33696297 +size 1788839 diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..4180463e19256eb61e8a494b85abf8fe94a14ceb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the person continues, they move further down the hallway, applying plaster to different sections of the ceiling. The camera angle shifts slightly, revealing more of the hallway and the partially painted walls. The person uses the putty knife to spread the plaster evenly, occasionally pausing to scrape excess material from the knife. The fisheye effect continues to alter the perceived size and shape of the putty knife and the surrounding environment.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1b435bc30c72df99c1db6bb1247d995ebfe06b57 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e18555fc61417cb326b8911a4e31bb5b7c788c754e6d39e37beae19f666bcad5 +size 1866371 diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..1e323149ef7b880ec9cbef7a159f0d4674661a22 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person progresses further down the hallway, focusing on reaching higher sections of the ceiling. The camera captures their movements as they apply plaster to the textured surface, ensuring even coverage. The fisheye lens distorts the view, making the putty knife appear larger and closer. The hallway has a white door and a window visible in the background.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9821271500e910d978a10057b2f276b70540b87e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e3f954986987896e1e50f5b91253385162f7ac8ad26f99bb2f83cc180be1f7d +size 1352907 diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..35abc003e6a6d8a09dbfaf8c483da9c681ddab45 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In this segment, the person continues to apply plaster to the ceiling, moving towards the end of the hallway. The camera angle shifts to show more of the hallway and the partially painted walls. The person uses the putty knife to spread the plaster evenly, occasionally pausing to scrape excess material from the knife. The fisheye effect continues to alter the perceived size and shape of the putty knife and the surrounding environment.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..56e691c1b08705cfc9d97039dae879bd60f9b2f0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42e3e2104e01b7552a4f6cabafebe60e6f09810a807b6ab832f91eb1890c80b9 +size 1245915 diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..de20d294c1791f2b2da0df9c15cd038705d953e2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with the person nearing the end of the hallway, continuing to apply plaster to the ceiling. The camera captures their movements as they reach higher sections of the ceiling, ensuring even coverage. The fisheye lens provides a distorted view of the surroundings, including a white door and a window. The person occasionally steps down to reload the putty knife with more plaster from the container.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d3e597863c39f34f28e284a08a3e5d590886636 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d48371b0de420dbb582a7d0f5c141381c7bfde2a2aa1e3191e4dd61ce02d1c80 +size 1714473 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..600384fe43749d66da15cb2dfdea09e9f1305289 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/console.log b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/console.log new file mode 100644 index 0000000000000000000000000000000000000000..648ed290eb29f1e83f81d0e2f46e451a98b51049 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/console.log @@ -0,0 +1,22 @@ +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/config.yaml +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.json +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:52:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:52:43|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/debug.log b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5a664a392d2e7b4f54955aad6b1972e5bf0059b6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/debug.log @@ -0,0 +1,353 @@ +[02-20 14:38:37|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 14:38:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:38:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 14:38:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 14:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 14:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 14:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 14:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 14:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 14:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 14:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 14:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 14:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 14:38:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 14:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 14:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 14:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 14:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 14:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 14:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 14:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 14:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 14:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 14:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 14:38:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 14:38:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 14:38:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 14:38:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 14:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 14:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 14:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 14:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 14:38:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 14:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 14:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 14:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 14:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:38:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 14:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 14:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:39:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 14:39:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 14:39:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.98 s +[02-20 14:39:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:39:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:39:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/config.yaml +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 14:39:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final part of the video, the player is fully immersed in playing the clarinet, their fingers moving swiftly and accurately over the keys. The camera maintains its first-person perspective, focusing on the hands and the instrument. The smartphone on the dresser continues to display the music app, and the player makes a final interaction with it before resuming their performance. The video ends with the player deeply engaged in their music, the clarinet as the central element of the scene.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.json +[02-20 14:39:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:39:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.mp4 +[02-20 14:39:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 14:39:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 14:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 14:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 14:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 14:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:52:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:52:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 14:52:43|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2001d8bc7d405a584c19bce0de4f11763d22c5d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a close-up of the player's hands holding a silver clarinet, positioned horizontally. The clarinet's black mouthpiece and reflective silver keys are prominent. The player, wearing a light gray shirt and black pants, sits cross-legged on the floor. In front of them is a music stand with sheet music for a 'Concerto in D Major.' To the left, a smartphone on a dark wooden dresser displays a music app, which the player taps to control playback. The background includes a bed with light blue bedding and a saxophone leaning against the wall.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92c848765709e8ebf42c0eedf88230245b9d0d59 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:590bef58529d73e411b1ebd9584fe78bf962e5ecc5035ef5929a5e398de20230 +size 875149 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9abb82abc5935bfe0bf0d12a82a576e6a4d33d6d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the initial setup, the player begins to play the clarinet, their fingers moving over the keys with precision. The camera remains focused on the hands and the instrument, capturing the intricate finger movements. The smartphone on the dresser continues to show the music app, and the player occasionally interacts with it, possibly adjusting the playback or checking the progress. The bed and saxophone remain in the background, providing context to the setting.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73bbe495d0f6fe6d43d244f886d87e30e512c3c9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34ace9c5b95af10139c7af1c8d8c8a8e5384e74453449aec76160290a93b3331 +size 902981 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e3ece1c3b921b78ccf4b11a092a890f87f30a41f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"As the video progresses, the player maintains a steady rhythm, their fingers fluidly transitioning between the keys of the clarinet. The camera angle stays consistent, emphasizing the detailed hand movements and the reflective surface of the instrument. The smartphone on the dresser is still visible, displaying the music app, and the player makes a few more taps or swipes on the screen. The background elements, including the bed and saxophone, remain unchanged, reinforcing the intimate and focused nature of the performance.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..660801349df91bf28537f176fa0b26a92c792d46 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8669f38ead6cd781de6d215853d27d13f90d85dd1e2eb0a6dd03cceb60759fd0 +size 775083 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..8a84153c05bb701c842114d87f5820562dbfc8a7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The player continues to play the clarinet, their hands moving with practiced ease over the keys. The camera captures the subtle nuances of the performance, highlighting the interaction between the fingers and the instrument. The smartphone on the dresser remains active, showing the music app, and the player occasionally checks or adjusts it. The bed with light blue bedding and the saxophone in the background provide a serene and personal atmosphere to the scene.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6a5c06993ce861794be0d46eab21719f3b4aa31f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e8be23efc19db4a9e098b9926c343ac7dbfa6593778a0b9eb86dddb89bc2472 +size 935276 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..848ed2eaf40f3547c7ae5c38162f491f83119d26 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final part of the video, the player is fully immersed in playing the clarinet, their fingers moving swiftly and accurately over the keys. The camera maintains its first-person perspective, focusing on the hands and the instrument. The smartphone on the dresser continues to display the music app, and the player makes a final interaction with it before resuming their performance. The video ends with the player deeply engaged in their music, the clarinet as the central element of the scene.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6f519fcabe67b07a42cc65e8bd8b78954e272849 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e31f510988d4101ccc793e57f2934ff420598526c9139522addea566e31938e7 +size 1222961 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad055d95645b7053bb68ad79ca7c1d41c2394511 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/console.log b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/console.log new file mode 100644 index 0000000000000000000000000000000000000000..7a7180332a24ce19fe5d4b3bbe584a7410063383 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/console.log @@ -0,0 +1,22 @@ +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:38:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:38:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:39:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:39:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/config.yaml +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.json +[02-20 08:39:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:53:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:53:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:53:58|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/debug.log b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b67eb61b51fa7d610159a1f954545f4008b2ca8c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/debug.log @@ -0,0 +1,353 @@ +[02-20 08:38:45|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 08:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 08:38:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 08:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 08:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 08:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 08:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 08:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 08:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 08:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 08:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 08:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 08:38:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 08:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 08:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 08:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 08:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 08:38:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 08:38:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 08:38:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 08:38:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 08:38:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 08:38:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 08:38:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 08:38:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 08:38:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 08:38:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 08:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 08:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 08:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 08:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 08:38:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 08:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 08:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 08:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:38:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:38:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 08:38:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:38:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 08:38:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:38:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:39:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 08:39:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:39:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 08:39:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 08:39:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 08:39:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 08:39:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:39:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 08:39:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 08:39:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 08:39:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:39:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:39:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 08:39:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 08:39:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.60 s +[02-20 08:39:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:39:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:39:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 08:39:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/config.yaml +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:39:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the person completes another envelope and places it next to the first one. They smooth out the envelopes and arrange them neatly on the table. The video concludes with the person preparing to start another envelope, continuing the crafting process.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.json +[02-20 08:39:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:39:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.mp4 +[02-20 08:39:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 08:39:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 08:39:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 08:39:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 08:53:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 08:53:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:53:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 08:53:58|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f26927c3a4c2028f1a9abd5a1da7e527453218e3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up view of a person's hands holding a rectangular sheet of light green paper. They use scissors to trim the edges of the paper, ensuring it is perfectly square. The person then folds the paper in half lengthwise, creating a central crease. Next, they fold the top corners diagonally inward to form triangular flaps at one end of the paper.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5bf4298fb7df2c0168bcc7a72cdfeb2fba5dd8a0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d35602704a76faec943c6fe77c3c9e8cdd9d65d931430adae714f242bfbe71e8 +size 1034694 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..7d5f142e273ff60bde8d965625b3b9ab2988569c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the previous scene, the person folds the bottom corners upward to meet the top flaps, forming a pointed end for the envelope. They then fold the paper further, bringing the sides together to create the main body of the envelope. The person smooths out the folds to ensure they are crisp and even. A colorful picture is placed on the table nearby, ready to be inserted into the envelope.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b23363bc14c599076add9559b99621c1a11aefcf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb0f5bcae134a7241e7587bbef7b7ba4a8c4d12e86ef3c81c8008ecc8fcf9a19 +size 1107223 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..615bf423b52bb7347a7c2ad5e5972a865ccc2457 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person opens the envelope slightly and carefully inserts the colorful picture inside, adjusting it to fit neatly within the envelope. They then fold the remaining flaps inward and secure them, completing the envelope. The person smooths out the envelope to ensure it is properly sealed and neat.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..86448d54bac9c42b93430048d1daefcfb6e6cbc1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d516748514c01ef12e954da582bbbe0a72dd642b502806b5ce7a2b08f51d218c +size 1180356 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..8a8074483a8a8d929b237b7cbffce7f9cef7fc22 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Next, the person lifts the finished envelope, displaying it briefly to show the completed craft. They then place the envelope aside and begin preparing another sheet of light green paper, repeating the process of trimming, folding, and sealing. The camera captures various angles as they work, highlighting the precision of their movements.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31f95af518c73cffcfd19e6812d7d5f0fa1059f8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69def85b0a066b10ef0128162a9dc351e541b11152feb709babb66e348acadc0 +size 1436251 diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..4f49e4ba5ef6adfb3d6436d9cd29c8da2fd72feb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person completes another envelope and places it next to the first one. They smooth out the envelopes and arrange them neatly on the table. The video concludes with the person preparing to start another envelope, continuing the crafting process.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d795df0537cf573c7d5244bd296cef6d9d2e935 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d798bb163802e9bfbe6c27fe24132f76aad4087be11d30ca21390f114e3af6c4 +size 1145835 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e15988c77497f054341646b8115ac02adef34f92 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/console.log b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c854c0ee4835bf637b4f91141b3415af3299ca69 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/console.log @@ -0,0 +1,22 @@ +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:56:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:56:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/config.yaml +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.json +[02-20 19:56:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 20:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 20:11:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 20:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 20:11:04|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/debug.log b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b11efd9bb99fd3163ccb283f90ef4cc5d667afd2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/debug.log @@ -0,0 +1,353 @@ +[02-20 19:55:46|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 19:55:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:55:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 19:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 19:55:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 19:55:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 19:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 19:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 19:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 19:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 19:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 19:55:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 19:55:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 19:55:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 19:55:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 19:55:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 19:55:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 19:55:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 19:55:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 19:55:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 19:55:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 19:55:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 19:55:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 19:55:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 19:55:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 19:55:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 19:55:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 19:55:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 19:55:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 19:55:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 19:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 19:56:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:56:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 19:56:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:56:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 19:56:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 19:56:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 19:56:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 19:56:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 19:56:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 19:56:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 19:56:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 19:56:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.06 s +[02-20 19:56:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 19:56:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 19:56:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/config.yaml +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 19:56:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the person makes final touches to the wooden post, ensuring all areas are covered. They lift the brush and step back slightly, revealing the freshly painted post. The camera shows the completed work, with the post looking smooth and evenly coated. The person then prepares to move on to the next section or clean up the workspace.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.json +[02-20 19:56:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 19:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.mp4 +[02-20 19:56:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 19:56:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 19:56:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 19:56:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 20:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 20:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 20:11:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 20:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 20:11:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 20:11:04|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3a8613c377ab14db2d4dbd69644e92f16084e17a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with the person beginning to paint a wooden post on a balcony. They are holding a yellow-handled paintbrush and dipping it into a black paint can placed on the tiled floor. The person starts applying paint in vertical strokes, moving around the post to cover all sides. The surrounding area includes green bushes and trees, and a plastic sheet is laid out on the floor to protect it from paint.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14af5b396a56e63fa8057841d9f21e5893718d5e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:392ec8cc4a97d8b3fd03b3563834b37b4c837083913a2d60a8734a485c56488f +size 1233993 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..26ef465411d00b98f37ef0551fc6d6d7b74e77d4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the person continues, they focus on different angles and sections of the wooden post. The camera captures close-up views of the brush applying paint, highlighting the texture of the wood and the smoothness of the strokes. The person occasionally adjusts their position and the plastic sheet on the floor to ensure even coverage. The background remains consistent with lush greenery and a clear view of the outdoor environment.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..494a56fd9b48db38a354872eb316d7076c0b322d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7081b0e3381bd83a8e9862552f8dc62c2c9f471feccd2575a765305072862b73 +size 1870040 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..f4ffc4eeedb008db874b2d1953442752ecd76c17 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person then moves to the top of the wooden post, making sure to cover all edges and corners. The camera angle shifts to show the top of the post and the surrounding railing. The person continues to apply paint in precise strokes, ensuring complete coverage. The plastic sheet on the floor is adjusted to accommodate the new position.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..89d033c02a0dc673be2e1e01d2811b15cfa56a99 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15e6d38a17cafdda0b2c68ef64fefe095a3ebabe2084025a7ef6ebdca1da0029 +size 1566588 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..d36b6ba1acd998aa263cf247dbed381094194e80 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the next segment, the person focuses on the lower sections of the post, making sure no spots are missed. The camera captures the detailed work, showing the brush moving smoothly over the wood. The person occasionally steps back to assess their progress, ensuring an even coat of paint. The background remains the same, with greenery and a clear view of the outdoor setting.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5052162f4f7c9c3efc688e7213062a0d2246394 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:088c7cda5f638f6aa1e3d944e21255564310c93156796fd7e8a3183fb0272db0 +size 1575807 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..a3d13597ed188f9ba323ee3d659f23f8a4ba0bfe --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person makes final touches to the wooden post, ensuring all areas are covered. They lift the brush and step back slightly, revealing the freshly painted post. The camera shows the completed work, with the post looking smooth and evenly coated. The person then prepares to move on to the next section or clean up the workspace.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..549ce766eadcecb52edd55b35dbf40a5d7e0a3cb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5688ecb65e930fa8525977e3750f318d853c103732348da7933b885d28c661e +size 1748984 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91fda47f796c448aa81cea12d0c5d70980087be2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/console.log b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/console.log new file mode 100644 index 0000000000000000000000000000000000000000..46bd77845a2e8c81a6dc23594cb9d7fd88fe1c50 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/console.log @@ -0,0 +1,22 @@ +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/config.yaml +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.json +[02-20 02:02:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:15:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:15:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/debug.log b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7e6c7f6bbb2d59193220051290a3b714ac660d86 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/debug.log @@ -0,0 +1,353 @@ +[02-20 02:01:33|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 02:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:01:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:01:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 02:01:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 02:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 02:01:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 02:01:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 02:01:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 02:01:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 02:01:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 02:01:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 02:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 02:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 02:01:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 02:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 02:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 02:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 02:01:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 02:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 02:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 02:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 02:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 02:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 02:01:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 02:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 02:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 02:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 02:01:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 02:01:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 02:01:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 02:01:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 02:01:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 02:01:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 02:01:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 02:01:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:01:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:01:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 02:01:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:01:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 02:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:01:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 02:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 02:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.16 s +[02-20 02:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 02:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 02:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 02:01:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:01:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:01:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:02:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 02:02:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 02:02:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.42 s +[02-20 02:02:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:02:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:02:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 02:02:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/config.yaml +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 02:02:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The video concludes with a focus on the basket's near-completion stage. The camera captures the artisan's hands as they make final adjustments to the weave, ensuring the basket is sturdy and well-formed. The video includes a mix of close-up and overhead shots, showcasing the intricate details and the artisan's expertise. The background remains consistent, with other artisans working on their projects and bamboo materials scattered around the workshop, highlighting the traditional and communal nature of the craft." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.json +[02-20 02:02:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:02:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.mp4 +[02-20 02:02:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 02:02:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 02:02:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 02:02:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 02:15:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:15:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:15:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:15:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 02:15:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..4714cdcbd4adb7eb4d4f430d7d3a6cf68dd9758f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a close-up shot of a partially completed bamboo basket, highlighting the vertical staves and the initial stages of horizontal weaving. The artisan's hands are seen carefully placing and securing the bamboo strips, using a tool to ensure the weave is tight and even. The camera captures the texture and natural colors of the bamboo, emphasizing the craftsmanship involved. The background reveals a rustic workshop setting with other baskets and materials visible.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e23cd1ef2b5d725ac27ee7a106ed7ccb67d7b895 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f086a49527a13c19dfeb7a5bbb536f51a9677a84d448ad5874d2dd5adaeb4f7b +size 1060277 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b556507b7f691286637c20d06fd1cdc65a94da75 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the camera shifts to a wider angle, showing the basket in a more complete state with a flared top. The artisan continues to weave the horizontal strips around the vertical staves, demonstrating precision and skill. The perspective changes to an overhead view, providing a clear look at the basket's structure and the intricate pattern being formed. In the background, other individuals are seen working on similar craft projects, adding to the communal atmosphere of the workshop.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63db3b494320681e0334d53c16544953c659f1c5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3f30950252f801cb73761c3401d06b558bacf744f07493a54e24742aeb4d6a4 +size 1962794 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..52369fa6a0c33601ca8f0f8a4905851cb9749da9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The next segment focuses on the detailed weaving process, with the camera capturing close-up shots of the artisan's hands as they meticulously interlace the bamboo strips. The basket's cylindrical shape and flared top become more defined, showcasing the progress made. The camera occasionally shifts to side views, highlighting the basket's form and the artisan's methodical approach. The background remains consistent, with other artisans continuing their work amidst the bamboo materials.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..97d43db99dbd0a2c2ab135869e2af81edc1bc19f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b146a503667f45a89185fdf200d707e0c3ca41145641d404bf2940dbe70dde49 +size 1838945 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..bcafe6009f0ac1ae401a5c6f5ae3fcf2c72a7ffb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In this part of the video, the camera provides a dynamic view of the basket-weaving process. The artisan's hands are shown making adjustments to the basket's position, ensuring the weave is even and secure. The video includes a mix of close-up and side views, emphasizing the texture and color variations of the bamboo. The background reveals more of the workshop, with other individuals engaged in similar craftwork, contributing to the overall ambiance of the scene.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..267293493f230439e255dfe354567336c9956676 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b04ea6794b6dd8a71365ff8cfe6c2e8bf624cf5dc3efa84389809648695ec2b +size 1731348 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..14504b461a150e188ddd1e3cef9dd36316f4258f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with a focus on the basket's near-completion stage. The camera captures the artisan's hands as they make final adjustments to the weave, ensuring the basket is sturdy and well-formed. The video includes a mix of close-up and overhead shots, showcasing the intricate details and the artisan's expertise. The background remains consistent, with other artisans working on their projects and bamboo materials scattered around the workshop, highlighting the traditional and communal nature of the craft.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..611269a30ca0c9c4c6a3e880274aa00ac2eb0495 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ef93ef7d8b1e136795c5df5f6327d7cd85acfd8f3380ce4250d402620cc1f39 +size 1904097 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c38031bed0277b01b5d5f6e8e8c6d1964dfaa31 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/console.log b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/console.log new file mode 100644 index 0000000000000000000000000000000000000000..5fcf761ff70c4798010120e43123ee055503e074 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/console.log @@ -0,0 +1,22 @@ +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 20:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 20:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/config.yaml +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.json +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 20:40:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 20:40:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 20:40:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 20:40:34|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/debug.log b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ddd595671d57ebbff52901f73c77a524a5baaa2d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/debug.log @@ -0,0 +1,353 @@ +[02-20 20:26:24|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 20:26:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 20:26:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 20:26:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 20:26:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 20:26:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 20:26:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 20:26:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 20:26:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 20:26:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 20:26:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 20:26:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 20:26:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 20:26:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 20:26:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 20:26:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 20:26:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 20:26:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 20:26:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 20:26:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 20:26:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 20:26:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 20:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 20:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 20:26:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 20:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 20:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 20:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 20:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 20:26:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 20:26:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 20:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 20:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 20:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 20:26:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 20:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 20:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 20:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 20:26:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:26:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 20:26:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 20:26:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:27:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 20:27:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 20:27:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.42 s +[02-20 20:27:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 20:27:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:27:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/config.yaml +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 20:27:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the person continues reading the book while the cat moves around the couch, eventually settling down near the person. The camera captures the cozy living room environment, showing the coffee table, TV, and other furniture. The video concludes with the person still engaged in reading, the cat comfortably resting on the couch, and the overall atmosphere remaining calm and relaxed.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.json +[02-20 20:27:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 20:27:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.mp4 +[02-20 20:27:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 20:27:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 20:27:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 20:27:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 20:40:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 20:40:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 20:40:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 20:40:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 20:40:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 20:40:34|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a305f9843095e0b7f50389f1d56dace1599a4380 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a first-person perspective of someone lying on a brown couch, reading a book. The book has white pages, black text, a blue sticky note on the left page, and colorful sticky notes on the right margin. The person is wearing a dark t-shirt, gray leggings, and a red string bracelet. They are holding the book on a patterned pillow in their lap. In the background, there is a black coffee table, a TV on a stand, and a dining area. The person occasionally shifts their position and adjusts the book while reading.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ce69a987589e9d6b5106ee3414b9e4a6fd5a513d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4493d1c95463912cc86e035724bdafb7b23554ca9a418c4ce9fbd19cdeab61a +size 1106300 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5d1c3a5429ccc628b888377550aa8cc1e4706d63 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the person continues reading, they make minor adjustments to their posture and the book. The camera angle remains consistent, showing the living room setting. The person briefly uses a smartphone, which is placed on the couch, before putting it aside. A white and orange cat enters the scene, walking across the coffee table and approaching the person on the couch.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7fd8018ace8cb80bcee9d1006bbcab382b28c2b8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d910afd526ff0ed198c98d0289575f56dc91bc59f10185561f45ce92aedbafef +size 1161930 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..8570e2a60ed14dd715f45058212ec2e24ef8f97a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person interacts with the cat by petting it while continuing to hold the book. The cat sits beside the person on the couch, appearing comfortable. The person then reaches for a black backpack that was on the couch, moves it to the floor, and adjusts the pillows around them. The cat remains on the couch, observing the surroundings.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2d5c4884a09d65c4669a896b676d51e31d37bd21 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a9f3beb57778377ba92c5f2a83d39c521a140f6d02f18e303241c4b96ac860 +size 1604711 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4e97a53a14381a3b1c571de42c7df6f3cecb5140 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The person picks up the book again and resumes reading. The cat moves around the couch, eventually settling down near the person. The camera angle shifts slightly, revealing more of the living room, including a bed and additional seating in the background. The person continues to read, occasionally glancing at the cat.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33b0d6a7e587d183504969338864951e04cb75b9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9d766492cb79569cb69154f94049537e0a0c27e10dcf12b63312d5fc8151e5 +size 1308151 diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..14b1e808f7c5810686fed3987a064fa362db595a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the person continues reading the book while the cat moves around the couch, eventually settling down near the person. The camera captures the cozy living room environment, showing the coffee table, TV, and other furniture. The video concludes with the person still engaged in reading, the cat comfortably resting on the couch, and the overall atmosphere remaining calm and relaxed.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b49c3d1222b76af174eeace8626cd03741d90599 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26f4c6cfcd9d414c446abe7c6f89a99fc0defd3a7b117b4b1ae9a06fdb858b1b +size 1274525 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0bd4fbb9f92a40d178e59880ef1649410aa7aec --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/console.log b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/console.log new file mode 100644 index 0000000000000000000000000000000000000000..24f44fd7ccdd9b5ce3665f08ac6a28f02d78416b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/console.log @@ -0,0 +1,22 @@ +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:06:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:06:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/config.yaml +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.json +[02-20 01:07:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:20:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:20:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:20:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:20:56|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/debug.log b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dc1946cc521266255720da06f7d14501c5556df6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/debug.log @@ -0,0 +1,353 @@ +[02-20 01:06:44|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 01:06:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:06:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 01:06:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 01:06:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 01:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 01:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 01:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 01:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 01:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 01:06:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 01:06:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 01:06:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 01:06:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 01:06:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 01:06:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 01:06:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 01:06:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 01:06:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 01:06:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 01:06:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 01:06:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 01:06:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 01:06:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 01:06:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:06:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:06:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 01:06:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:06:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:06:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:06:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 01:06:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:06:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 01:06:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:06:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:06:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 01:06:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:07:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:07:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:07:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:07:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 01:07:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 01:07:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.14 s +[02-20 01:07:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:07:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:07:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/config.yaml +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 01:07:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final segment, the person returns an empty blue plastic egg tray holder to the metal rack. The camera captures the person's hands as they place the empty tray back onto the rack. The video concludes with a view of the organized trays on the ground and the metal rack, showcasing the completed cycle of handling the egg tray holders." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.json +[02-20 01:07:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:07:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.mp4 +[02-20 01:07:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 01:07:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 01:07:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 01:07:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 01:20:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 01:20:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:20:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:20:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:20:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 01:20:56|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..b8911d567a40abcee23930a97db1aebc5ef70a69 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a black screen, which quickly transitions to an outdoor setting. A person is seen holding a blue plastic egg tray holder with three white egg cartons on it. The person is wearing dark clothing and sandals, and their hands are visible as they handle the tray. The ground is made of large, square tiles, and the lighting suggests it is daytime with shadows cast on the ground.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..508470fc07f2849e12d88f43552b49eec3d54eaa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:512c7d4486c2a0d9733f8b2e4caf0268e6d48a1d1470f198ad603c1d4c5e8267 +size 914757 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..ea72a3ddb9a9dc7b1de5e014850a207372d81bf0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"The person walks towards a metal rack filled with similar blue plastic egg tray holders, each containing white egg cartons. The camera angle shifts to show the person picking up another tray from the rack. The focus is on the person's hands and the tray, with the background showing the tiled ground and the metal rack.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1952a249011688e11081434ba605586e5051fff4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:539c69d4982b17e8410407eb0373fbcbb6abfc85f8d4c008e2a54e1ef157988d +size 1622253 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..760f5f122d4041dd12cdc4eae87bbff24e220d78 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person carries the tray across the tiled outdoor area, and the camera captures various angles of the tray and the egg cartons. The person places the tray down on the ground, arranging it neatly among other trays already there. The camera provides close-up views of the egg cartons and the blue tray, emphasizing the details of the objects.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8c90ac2215e2061e66e660e8aa8b08c04ae654a0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a12997be262e9a92dfd127d0d97c544fbf0986a53e109812c22554edee5214d +size 1734778 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..05e583d4169bde60a113fdddede1d56ef3c09db8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The person continues to handle the trays, picking up another one from the metal rack. The camera follows the person's movements, showing the tray being carried and placed down. The background remains consistent with the tiled ground and the metal rack filled with trays. The person's actions are smooth and practiced, indicating familiarity with the task.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e4fabf914e4ababaa009c7e9d7648b37efeb33db --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:737982a54a3d7a8bd8740e4daee64f332ea379b702baea2e6abe94227ae741df +size 1960586 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..077ef9d73b8cde726c09f8ef4eadee0d29d9cda9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the person returns an empty blue plastic egg tray holder to the metal rack. The camera captures the person's hands as they place the empty tray back onto the rack. The video concludes with a view of the organized trays on the ground and the metal rack, showcasing the completed cycle of handling the egg tray holders.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1ece4363a6ba9c979f6cfff5d188e44efa9cdabb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:775db7be2fb9fd2158dac67748146b0b7e733d599939a30efd67c5b2c125c458 +size 1426702 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..558089d205135e7da236c15f3ed185cafa60af82 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/console.log b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/console.log new file mode 100644 index 0000000000000000000000000000000000000000..9bcb851a2c3d7d89cf4c04ef8480c18865fff50d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/console.log @@ -0,0 +1,22 @@ +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:04:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:04:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/config.yaml +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.json +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:18:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:18:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:18:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:18:26|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/debug.log b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..67c5f0bfce8e9a23fc22f4e696a33a056115d160 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/debug.log @@ -0,0 +1,353 @@ +[02-20 07:04:17|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 07:04:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:04:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 07:04:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 07:04:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 07:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 07:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 07:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 07:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 07:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 07:04:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 07:04:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 07:04:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 07:04:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 07:04:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 07:04:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 07:04:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 07:04:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 07:04:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 07:04:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 07:04:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 07:04:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 07:04:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 07:04:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 07:04:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 07:04:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:04:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:04:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 07:04:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:04:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:04:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:04:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 07:04:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:04:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 07:04:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:04:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:04:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 07:04:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:04:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:04:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:04:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 07:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 07:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.56 s +[02-20 07:04:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:05:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:05:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/config.yaml +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 07:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The video concludes with the artist making final adjustments to the cloud painting, using both the green and red-handled brushes to perfect the details. The camera provides close-up shots of the brushwork, emphasizing the artist's skill and dedication to creating a lifelike cloud scene. The surrounding environment remains consistent, with the patterned rug, wooden floor, and cluttered table filled with art supplies visible throughout." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.json +[02-20 07:05:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.mp4 +[02-20 07:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 07:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 07:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 07:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 07:18:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 07:18:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:18:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:18:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:18:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 07:18:26|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..21f9a056ac11625ae32652be3dc1b302ab653470 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up view of the artist's hand holding a green-handled paintbrush, applying soft blue and pink strokes to a canvas depicting a cloud-filled sky. The artist, dressed in a camouflage shirt and black pants, is seated on a chair with a patterned rug beneath them. The initial frames focus on the delicate brushwork as the artist begins to form the base of the cloud scene.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68e7ac48a90da3e5c643ce4d9066858bdf5443ab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc5f042485f5b104eda64843aa1ec43041d76209beaa0142d6a58eaf14255944 +size 693731 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..7b3f02de03ffa3be6f0b3505ba07af298115a9b2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the artist continues to build upon the cloud painting, adding layers of color and texture. The camera captures the artist's movements as they occasionally glance at a laptop displaying a reference image of clouds. The surrounding area includes a wooden floor and a table cluttered with various art supplies, indicating an active and creative workspace.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0079c66c885360a8e7b9f2a1df93dc43d3ed12d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f383d21c25946ac9facd1cc80d6e8e337b75a5562ef51b5857cc0a9c18059ff +size 1094485 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..7514246ac73bc78a9c8675641aeb08bd151ae2ee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The artist then shifts their focus to adding more depth and dimension to the clouds, using the green-handled brush to blend and layer colors effectively. The camera angle changes to provide a wider view of the workspace, showing the artist's interaction with the reference image on the laptop and their meticulous attention to detail.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc9ea67bfe6ed7e5e88d12e17f1bb4b71fce9172 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc4d09b8e21593c60980b0007798f2f7430e9267b6db4d6244f38c3ce12d3a8b +size 1036085 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..2f8f03462e797cd43960c7d362c69ce065334925 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the next segment, the artist picks up a red-handled brush to add finer details and highlights to the cloud painting. The camera zooms in on the brushwork, capturing the intricate blending of colors and the creation of realistic cloud textures. The artist's movements are deliberate and precise, enhancing the overall composition of the artwork.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf5e9a942d8cada4952dde075405312ce8b9e13d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e184b2fdf8e7cd9f29a7c331b14675495487729d2c4ae593512bc3e5d0cfd23 +size 1464545 diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..13646f2258b29788080824884bb5428706dcb167 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with the artist making final adjustments to the cloud painting, using both the green and red-handled brushes to perfect the details. The camera provides close-up shots of the brushwork, emphasizing the artist's skill and dedication to creating a lifelike cloud scene. The surrounding environment remains consistent, with the patterned rug, wooden floor, and cluttered table filled with art supplies visible throughout.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4fce7a50aaa30a43930bc1cd592477aaaa8367d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8126c236e6bd395d8db8031e555b8b0d3dc1b0f01090d46e242bcc49bc3d036f +size 1821797 diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b93e1efba3b8468db1d5b9ee2dda42a41662910c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/console.log b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ecb44b3ce4cff94dec8c68f02ea92f295b552432 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/console.log @@ -0,0 +1,22 @@ +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 23:07:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 23:07:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/config.yaml +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.json +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 23:21:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 23:21:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 23:21:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 23:21:25|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/debug.log b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6bed527ba8184fc3610f79596f8473b888d2b6f2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/debug.log @@ -0,0 +1,353 @@ +[02-20 23:07:14|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 23:07:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 23:07:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 23:07:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 23:07:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 23:07:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 23:07:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 23:07:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 23:07:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 23:07:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 23:07:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 23:07:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 23:07:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 23:07:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 23:07:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 23:07:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 23:07:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 23:07:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 23:07:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 23:07:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 23:07:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 23:07:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 23:07:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 23:07:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 23:07:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 23:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 23:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 23:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 23:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 23:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 23:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 23:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 23:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 23:07:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 23:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 23:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 23:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 23:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 23:07:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 23:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 23:07:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 23:07:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 23:07:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 23:07:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 23:07:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 23:07:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 23:07:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 23:07:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 23:07:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 23:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 23:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 23:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.14 s +[02-20 23:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 23:08:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 23:08:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/config.yaml +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 23:08:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='To finish, the person washes the knife under running water in the sink, ensuring it is clean before returning it to its place. They also dispose of any packaging from the spreads. The video ends with the person making final adjustments to the presentation of the bread slices on the plate, completing the preparation process.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.json +[02-20 23:08:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 23:08:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.mp4 +[02-20 23:08:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 23:08:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 23:08:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 23:08:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 23:21:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 23:21:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 23:21:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 23:21:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 23:21:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 23:21:25|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a886456079f6e4ba66f9fdd62b1e01ce62d7564d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a first-person view of a kitchen counter cluttered with appliances like a coffee maker, kettle, and toaster. Two golden-brown toasted bread slices are visible on a white cutting board. The person reaches into the oven, which is integrated into the wooden countertop, and carefully pulls out the hot bread using a striped cloth.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5f8171be50a7e6d17241c38147247ce1b57bce70 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85207c17402bfabfecc29a2b554520bad1be4a279b184a0404b633a372895b6c +size 1475528 diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..3c5420aa8936512a7f0b25794a918fd50466561a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"After placing the toasted bread slices on the cutting board, the person picks up a stick of butter and spreads it evenly on one of the slices. The camera captures their hands as they work, showing the texture of the butter melting slightly on the warm bread. The other slice remains plain for now.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b80e941301a0447575f8f9d9eb3cdebc1bc40300 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18ee066f345e8e0b37d6c5a9f134a3d3b3f61d1f952ea90e859418d82be184c8 +size 1606258 diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e2075d7db18c665e26386196df4cf07e7d7d71c8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Next, the person retrieves a small container of jam from the counter and uses a knife to spread it over the buttered slice. The vibrant red color of the jam contrasts with the golden-brown bread. Meanwhile, the second slice waits to be prepared.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87adfcd9b22fdcb048b80382e6819d119dcb0487 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c619a798760fdc7a0acda92ee612f2464383598f323e4a21acf4b9474517b0f5 +size 1244433 diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..25a3ed902f9e066ff578d20947e220d168917a29 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The person then takes a chocolate spread from a nearby cabinet and applies it to the second slice of bread. They ensure an even layer, making sure the chocolate complements the texture of the toast. Once both slices are fully prepared, they arrange them neatly on a white plate.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9efb7b7e09e0a0556798904371943ce362b81003 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cb70d30277ff7e887b97b6db8dff6cf918a64fc71a3a7f31ddfd9f2ccc0a712 +size 1544958 diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..72dc40f84459c102cc80170c2fc955332df76744 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"To finish, the person washes the knife under running water in the sink, ensuring it is clean before returning it to its place. They also dispose of any packaging from the spreads. The video ends with the person making final adjustments to the presentation of the bread slices on the plate, completing the preparation process.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d306a62d5f491f72ea4140bde44479fb714815ee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20dca2310799e0043a7cfbe3c31bc7127c16f1083ab73a1073a47eb991f07872 +size 2137079 diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c35c20c4654c98128b32be041a78fff49b22ea40 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/console.log b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/console.log new file mode 100644 index 0000000000000000000000000000000000000000..5743113a2ea228bcb44113e0766d12362b7d4405 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/console.log @@ -0,0 +1,22 @@ +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:18:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:18:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/config.yaml +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.json +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:32:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:32:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:32:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:32:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/debug.log b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c718bf4cc7f547dc97964c98be223005590c9c28 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/debug.log @@ -0,0 +1,353 @@ +[02-20 06:17:57|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 06:18:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:18:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 06:18:06|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 06:18:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 06:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 06:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 06:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 06:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 06:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 06:18:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 06:18:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 06:18:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 06:18:06|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 06:18:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 06:18:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 06:18:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 06:18:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 06:18:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 06:18:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 06:18:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 06:18:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 06:18:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 06:18:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 06:18:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 06:18:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 06:18:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 06:18:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 06:18:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 06:18:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 06:18:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 06:18:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 06:18:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 06:18:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 06:18:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:18:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:18:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 06:18:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:18:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:18:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:18:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 06:18:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:18:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 06:18:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:18:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:18:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 06:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:18:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 06:18:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 06:18:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.00 s +[02-20 06:18:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:18:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:18:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/config.yaml +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 06:18:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the person finishes the embroidery by adding the last touches to the holly design. The overhead camera captures the final stitches, showcasing the completed section of the Christmas-themed fabric. The person then prepares to cut the thread, concluding the embroidery process. The video ends with a clear view of the nearly finished design, with the pattern guide and crafting materials still present on the table.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.json +[02-20 06:18:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.mp4 +[02-20 06:18:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 06:18:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 06:18:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 06:18:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 06:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 06:32:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:32:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:32:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:32:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 06:32:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e1abd4fa0926d9594adc2d44ee2e942783ee149e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a person beginning to embroider a Christmas design on a white fabric. The design includes a lantern, a star, and holly, and the person is seen stitching green holly leaves and red berries. The camera provides an overhead view, focusing on the hands and the fabric, while the person holds the fabric in their left hand and uses a needle with their right hand. A cross-stitch pattern guide is placed on the table, offering a reference for the design.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57c653240fc1e4f7294c37a907c10267e4afab8f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:916b5cef357968b007a24fc3d2bcb984f116d4d47a5b719792e48d30c250db53 +size 647388 diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9e08dfb52e4e67f700e794971bca26165ddee720 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the person continues to add more holly leaves and berries to the embroidery. The overhead camera angle remains consistent, capturing the detailed stitching process. The person occasionally shifts the fabric to work on different sections of the design, ensuring even coverage of the holly pattern. The floral tablecloth and the pattern guide are still visible, providing a stable backdrop for the embroidery activity.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..335507621a768f4845936a41f62e2173fbcd4cf7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8f75db22d521d65e7e30bf653254afdc25e439af441c4f058b57bb4d425d270 +size 1778635 diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..297f0a2497a231c5d5114b73b914e4c69a8cc679 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person then focuses on refining the holly design, adding finer details to the leaves and berries. The overhead camera captures the meticulous nature of the stitching, with the person's hands moving precisely to create the desired effect. The fabric is adjusted frequently to allow for better access to the embroidery area. The pattern guide remains a constant reference point throughout this segment.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a6ce90c3d3a18153c64f5697820f35e9a4a6ad79 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6b85faf13041dd8dbd11d8e9f471a8b8b3d24e9d5aefbe258858f8325ccee89 +size 1082528 diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..6afd83631a558743f2da07188aaff1acbb671ea4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Moving forward, the person continues to embellish the Christmas-themed embroidery, concentrating on completing the holly section. The overhead camera angle stays fixed, highlighting the intricate stitching work. The person occasionally glances at the pattern guide to ensure the design aligns with the intended pattern. The floral tablecloth and the surrounding crafting tools, such as scissors and additional threads, remain in view.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..197b9099ab786d82e614407b14ed68c508e2f83d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f98ab060b3a302abe4ee04740b9797288ae568fad0d0445350d052c780544b3 +size 1070101 diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..aeb64af2b6ee4d05ecc6af73f9e0cbb6fe3e084f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the person finishes the embroidery by adding the last touches to the holly design. The overhead camera captures the final stitches, showcasing the completed section of the Christmas-themed fabric. The person then prepares to cut the thread, concluding the embroidery process. The video ends with a clear view of the nearly finished design, with the pattern guide and crafting materials still present on the table.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3ea492bb163caaba4f2c3a8a0e91742f3be8f1f1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dab015668a3fe6054a8f5f4d4728611d95a852f939a16edfb8fced067e60760d +size 1135964 diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d95761bb05f80e564440eeb3ed2750ebc918db6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/console.log b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/console.log new file mode 100644 index 0000000000000000000000000000000000000000..0e411e7f7e3d8732eba0998272bf937f6f0a7127 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/console.log @@ -0,0 +1,22 @@ +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:03:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:03:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/config.yaml +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.json +[02-20 16:03:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:17:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:17:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:17:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:17:15|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/debug.log b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ac9db91d0d046cf8e78b51338766fd6dab07152d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/debug.log @@ -0,0 +1,353 @@ +[02-20 16:03:09|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 16:03:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:03:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:03:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 16:03:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 16:03:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 16:03:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 16:03:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 16:03:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 16:03:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 16:03:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 16:03:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 16:03:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 16:03:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 16:03:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 16:03:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 16:03:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 16:03:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 16:03:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 16:03:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 16:03:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 16:03:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 16:03:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 16:03:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 16:03:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 16:03:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 16:03:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:03:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:03:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 16:03:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:03:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:03:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:03:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 16:03:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:03:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 16:03:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:03:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:03:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 16:03:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:03:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.61 s +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.61 s +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:03:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:03:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:03:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 16:03:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 16:03:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.94 s +[02-20 16:03:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:03:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:03:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 16:03:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/config.yaml +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 16:03:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final moments, the mechanic completes the adjustments to the tie rod end, using the second wrench to finalize the task. They step back slightly, inspecting their work to ensure everything is properly secured. The video ends with the mechanic standing next to the vehicle, having successfully completed the repair of the tie rod end.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.json +[02-20 16:03:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.mp4 +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 16:03:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 16:03:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 16:03:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 16:17:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 16:17:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:17:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:17:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:17:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 16:17:15|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..25b73d902539b72744d1fe4b80dec23b4a6b5deb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens in a busy auto repair garage, showcasing a mechanic wearing a black shirt with green text. The mechanic walks across the concrete floor, passing by various tools and equipment, and approaches a blue toolbox. They open a drawer containing an assortment of tools and select a wrench, preparing for the task ahead.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9d8884d78899547deee4ab59910ee34f510e8b55 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:885b47b6eef01af5fc2eb326bb8540ddff31482ff8ece77f3f223cdaab368233 +size 1712153 diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..d7f640e1a7472e97dc0fd04f6bafaef6d940925b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Moving to a vehicle elevated on a lift, the mechanic focuses on the front suspension. The camera zooms in on a black cylindrical tie rod end, which has a metallic silver-colored clip attached to one end. The mechanic uses the wrench to begin loosening or tightening the tie rod end, which is connected to a larger rusty suspension component.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1dc199fa78b25982f4ebbe48cd4b3bc7b938a28a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ae106bf2128010fe3675638ebcb12309d125a0331a892ac5d1c35f1ae39db8b +size 1424328 diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..59931f76c1d1090970757618d2bfee9dabd54cde --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"After making initial adjustments, the mechanic steps away from the vehicle to retrieve a second wrench from a nearby red toolbox. The camera follows their movement as they walk back to the vehicle, holding both wrenches. The mechanic returns to the tie rod end, now ready to use the second wrench for further adjustments.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e927d6f9935de40e059bdbb7fab3b565fc8d2e1e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:920e845aefa57d61dabcc9befcc4fdebe8043925a370409e78f6cf707213272b +size 1612782 diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..7835f2894a30bc489d686254afbe549ecbc2a788 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The mechanic continues working on the tie rod end, alternating between the two wrenches to ensure the connection is secure. They carefully tighten the bolts, making precise adjustments to the tie rod end. The camera captures close-up shots of the mechanic's hands as they work, highlighting the detailed nature of the repair.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d54291e8ee756fce5ef8c6415b169862f8d6b500 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdf26544bf02bb31f2275e746710ec934c461cc504b65b0f59983f8007970ea2 +size 1283986 diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..82737fb703d6e3d297379f958ad3b7021f4a2993 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments, the mechanic completes the adjustments to the tie rod end, using the second wrench to finalize the task. They step back slightly, inspecting their work to ensure everything is properly secured. The video ends with the mechanic standing next to the vehicle, having successfully completed the repair of the tie rod end.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..793d4a689e6e11c5c541b91b3906fc7edb8458cc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e33959185d78fce6e35e267f8b261e8641ab403e4ca37eded788e67ec6b35ceb +size 1447702 diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80168aad05ba9451ddc90fbc32fdbd6e2aa48267 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/console.log b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c3139ca65d2b6c37be3f3b66ae3446dbcd5bf00b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/console.log @@ -0,0 +1,22 @@ +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/config.yaml +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.json +[02-20 06:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:18:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:18:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:18:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:18:43|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/debug.log b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a561efda10399349e0ae915e825799f50948b934 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/debug.log @@ -0,0 +1,353 @@ +[02-20 06:04:37|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 06:04:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:04:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:04:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 06:04:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 06:04:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 06:04:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 06:04:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 06:04:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 06:04:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 06:04:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 06:04:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 06:04:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 06:04:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 06:04:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 06:04:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 06:04:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 06:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 06:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 06:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 06:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 06:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 06:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 06:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 06:04:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 06:04:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 06:04:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 06:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 06:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 06:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 06:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 06:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 06:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 06:04:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 06:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 06:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 06:04:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:04:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 06:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:04:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 06:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:04:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:04:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 06:04:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:04:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.65 s +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.65 s +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.65 s +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:04:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:04:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:05:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 06:05:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 06:05:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.63 s +[02-20 06:05:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:05:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 06:05:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/config.yaml +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 06:05:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the person makes minor tweaks to the cylindrical component, ensuring it is perfectly aligned and secured within the machine. The camera provides a close-up view of the component, showcasing the precision and care taken during the process. The person steps back briefly, possibly to assess their work, before returning to make any necessary final adjustments. The video concludes with a detailed shot of the fully assembled component, highlighting the successful completion of the task.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.json +[02-20 06:05:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.mp4 +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 06:05:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 06:05:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 06:05:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 06:18:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 06:18:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:18:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:18:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:18:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 06:18:43|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..96243d67d4da27b3ee636c50354cb52a90c2a6e7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up shot of a person wearing blue gloves and a white lab coat, working on a large industrial machine. The main object of focus is a cylindrical metallic component with blue seals on both ends, which is part of a complex assembly involving numerous hoses and wires. The person begins by adjusting the component, ensuring it is properly positioned within the machine. The camera angle shifts slightly, providing a broader view of the workspace, which includes various tools and equipment scattered around.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f728142b4014c678aab5e69a2df87c6ef3c8cc47 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5cf1905c1a207b75ead9de8d86ffe15383a0de92a669144b114da17072a043 +size 1441800 diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..534ad0f0eea44e2f8f487f717dfa7aae8ac4a29b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the person continues to manipulate the cylindrical component, making precise adjustments to its alignment. The camera zooms in on the blue seals and the attached copper wires, highlighting the intricate details of the component. The person then picks up a smartphone, likely referencing technical specifications or settings related to the machine. The camera captures this action from a slightly different angle, showing the person's hands holding the phone while still working on the component.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2242291f36a3e02fa42c97b17da07156eb60b3ce --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fa06f0e759819e476d168f9059a039d035afe4325ab7714f84cfd704f8dc0ef +size 1434796 diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d9d69e38cebfe10e5df76df8b58b52143d93a2df --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Next, the person focuses on securing the cylindrical component within the machine. The camera provides a close-up view of the component's connection points, showing the person tightening bolts and ensuring all connections are secure. The background remains consistent with the industrial workshop setting, with various hoses and wires visible. The person then steps back to assess their work, making sure everything is properly aligned and functioning correctly.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cd670f22997b05f0ff5f5ea2d7222c171cc45802 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f81245687ef4e6fa9b56b81181a48bce6d5d438a778f58a551a2e5769e00f9d9 +size 1523666 diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4bcf9b497a52411e9b3235b272b2179becb57249 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The video continues with the person making final adjustments to the cylindrical component. The camera captures the component from multiple angles, emphasizing the blue seals and the metallic sheen of the cylinder. The person uses their hands to ensure all connections are tight and secure, paying close attention to the details. The background remains filled with various tools and equipment, reinforcing the industrial workshop environment.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fd10ae01e18e1cedb516efbdadafdf1bc592c35c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e700c59721ce23244c31a02025de74a8e46fefda71c38de0b2801e99a4a28b6b +size 1502015 diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c108eb4c76ea9a6162371dab1861b431fca5eb7f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the person makes minor tweaks to the cylindrical component, ensuring it is perfectly aligned and secured within the machine. The camera provides a close-up view of the component, showcasing the precision and care taken during the process. The person steps back briefly, possibly to assess their work, before returning to make any necessary final adjustments. The video concludes with a detailed shot of the fully assembled component, highlighting the successful completion of the task.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f62ae587dc30acf0aa9c4bda10a93509d058b6c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab56567e98e6a4d1f2902a249a63e7ed39a04745e07ef2f29c47a515418c558a +size 1140091 diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80163735df1d169ca57a5c08209b138206a8f75a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/console.log b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..2c55eef61c827021b6800d67f2d19821920118d3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/console.log @@ -0,0 +1,22 @@ +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/config.yaml +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.json +[02-20 21:53:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 22:06:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 22:06:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 22:06:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 22:06:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/debug.log b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ea3f3520ebd8972586a70c2369af0c1c5aa634d7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/debug.log @@ -0,0 +1,353 @@ +[02-20 21:52:25|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 21:52:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:52:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:52:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 21:52:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 21:52:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 21:52:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 21:52:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 21:52:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 21:52:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 21:52:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 21:52:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 21:52:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 21:52:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 21:52:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 21:52:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 21:52:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 21:52:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 21:52:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 21:52:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 21:52:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 21:52:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 21:52:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 21:52:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 21:52:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 21:52:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 21:52:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:52:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:52:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 21:52:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:52:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 21:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:52:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 21:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:52:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 21:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 21:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 21:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 21:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 21:52:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:52:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:52:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:52:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 21:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 21:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.35 s +[02-20 21:53:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:53:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:53:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 21:53:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/config.yaml +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 21:53:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the person makes the cut on the second plank using the miter saw. After completing the cut, they remove the piece and place it with the others. The person then steps back, assessing the progress of their work. The video concludes with them picking up another plank from the ground, ready to continue the task. The camera captures the entire process, highlighting the precision and care taken in handling the wooden planks.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.json +[02-20 21:53:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:53:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.mp4 +[02-20 21:53:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 21:53:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 21:53:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 21:53:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 22:06:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 22:06:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 22:06:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 22:06:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 22:06:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 22:06:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d22d6632e807164f140437eb1a13d17b9d21ed80 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with the person standing in a grassy outdoor area, surrounded by various tools and wooden planks. They pick up a dark brown wooden plank, which is rectangular with a smooth surface and visible wood grain. The person carries the plank towards a DeWalt miter saw, positioning it on the saw's base. The camera angle provides a close-up view of the plank and the person's hands as they handle it.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ade47e91a2819ca1ffc56cbc59253b9c640b4aff --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26a5a8804851eba194b641ccffc1086680c3ba53f24b640c4cccb4c4e6d5531a +size 1355931 diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..6544c57fee33cfcef257e79f8a40c46a1c41b6dd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Next, the person uses a yellow tape measure to measure the length of the plank while it rests on the miter saw. They hold the tape measure firmly against the wood, ensuring an accurate measurement. The camera captures this action from above, showing the person's tattooed arm and the details of the tape measure. After measuring, they adjust the plank on the saw, preparing it for the cut.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8ba3d3d8bfccd83e5bba40ecb88f9ae8b4d131b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c4aa3885bac7ac6d8b9fc1542d293d0b3ace43aa2e1624a5c91ebcca880790 +size 1489573 diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..6e431320e71aa5d857dc1e6ae252b7d008cc77f8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person then makes the cut on the plank using the miter saw. The saw blade spins rapidly as they guide the plank through the cut. Once the cut is complete, they remove the freshly cut piece and place it aside. The person then picks up another plank from the ground, repeating the process of measuring and positioning it on the saw. The camera follows their movements, capturing the entire sequence from different angles.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14db63b74d4f2eb3982d1c35a46788ccde56b6c1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff3adc4161dfd63985e13fa5d89c9f62386d578468847f2cbc3ee9863f8b3e9a +size 1620882 diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..629d3a043cc0bc704f037d1883b1be276120d36a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Continuing from the previous scene, the person measures the new plank with the tape measure, ensuring it is the correct length for the project. They adjust the plank on the miter saw, making sure it is properly aligned. The camera angle shifts slightly, providing a broader view of the workspace, which includes various tools and materials scattered around the grassy area.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e3d06a68aa93d6b4613689a794ca17b33918a921 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4cbb8f10fc18ca74a55c0c6a827049b9249483c30c813d7a3a6f1c87f578920 +size 1858665 diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..7ec84c281ff3bac9e1f89151a18d6333a7ad10cd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person makes the cut on the second plank using the miter saw. After completing the cut, they remove the piece and place it with the others. The person then steps back, assessing the progress of their work. The video concludes with them picking up another plank from the ground, ready to continue the task. The camera captures the entire process, highlighting the precision and care taken in handling the wooden planks.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab05ce9d8575bfbfeeb57b4aff89e7a6ceb53716 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:521b585fba0e0ae42dea3852acea2f657b2d4e305d9d8b82f8438066336d5253 +size 2019719 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcfbb14174eacbe75c7180d9a5ca854b314cebcd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/console.log b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c42ee13da01aac2185af201de9062dbe28eb779d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/console.log @@ -0,0 +1,22 @@ +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 09:38:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 09:38:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/config.yaml +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.json +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 09:52:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 09:52:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 09:52:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 09:52:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/debug.log b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..261c8542050535feecfdd4996dcf8d99195e76a5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/debug.log @@ -0,0 +1,353 @@ +[02-20 09:38:41|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 09:38:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 09:38:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 09:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 09:38:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 09:38:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 09:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 09:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 09:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 09:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 09:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 09:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 09:38:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 09:38:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 09:38:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 09:38:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 09:38:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 09:38:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 09:38:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 09:38:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 09:38:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 09:38:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 09:38:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 09:38:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 09:38:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 09:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 09:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 09:38:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 09:38:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 09:38:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 09:38:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 09:38:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 09:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 09:38:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 09:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 09:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 09:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 09:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 09:38:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 09:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 09:38:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 09:38:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 09:38:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 09:38:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 09:38:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 09:38:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.57 s +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:38:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 09:38:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 09:38:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:39:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 09:39:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 09:39:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.86 s +[02-20 09:39:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 09:39:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:39:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/config.yaml +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 09:39:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the person makes any last-minute adjustments to the sliced pizza, ensuring each piece is perfectly portioned and ready for serving. The camera focuses on the completed pizza, showcasing the even grid pattern and the appealing appearance of the cheese-topped slices. The person stands by, ready to serve the freshly cut pizza.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.json +[02-20 09:39:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 09:39:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.mp4 +[02-20 09:39:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 09:39:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 09:39:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 09:39:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 09:52:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 09:52:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 09:52:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 09:52:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 09:52:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 09:52:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..48390944301e0ac73d2ee42af18c1556fef5b492 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a person in a kitchen, wearing a white shirt, retrieving a freshly baked rectangular pizza from an oven. The pizza, topped with melted cheese and visible pepperoni and sausage, is placed on a metal baking tray. The person moves the tray to a cooling rack, allowing the pizza to rest for a moment. The camera captures various dynamic angles, focusing on the pizza and the person's actions.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78ba27a87598410e91c2beb97db0468de4a7e571 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f66714b40c84c351c82066887f325f0f7990f8acc5e38387e5061f5950aad25 +size 1555328 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..e6e16ea416c4b3e1c7ba341cee7534d585223d7c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the previous scene, the person begins to cut the rectangular pizza into even slices. Using a knife, they create a grid pattern across the surface of the pizza, ensuring each slice is uniform. The camera captures close-up views of the cutting process, highlighting the texture of the cheese and toppings. The person works methodically, dividing the pizza into multiple portions.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b31389e768c614421c41e7b174420123bbf9be65 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71ba447fe0bbedc67ef3b729d1f2bb2fc65c07ac8aeeb73899f5b30f469650d1 +size 1221634 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..8c9f8b2884b34df12438618ac264c4c42f1d53c8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person continues to cut the pizza, making precise movements with the knife to ensure each slice is perfectly portioned. The camera focuses on the detailed actions, showing the texture of the cheese and the even distribution of toppings. The person works efficiently, completing the division of the pizza into individual slices.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..71b067df18cf7d00130a103f3bec429fecd5f2d6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2417afc7ea32f149e8e1e133244c09bd8a36caa0f585efd365971a35d1d5f946 +size 1263715 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..86abb76eeaccff225d2c46b5712e9eee4f4cc044 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the cutting process nears completion, the person makes final adjustments to the slices, ensuring each piece is evenly sized and ready for serving. The camera captures the final touches, showing the neatly arranged slices on the metal baking tray. The person steps back, presenting the fully prepared pizza.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51545c502ac30386cef8b41649b0e7b2f9757d6c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cca6df958f6778ea8feb2d0b547bd802187f28b79cc46f1c41d396211d99f1fe +size 1563062 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..700b16a33f7e05d4ac71d814ca87a551ffce8e08 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person makes any last-minute adjustments to the sliced pizza, ensuring each piece is perfectly portioned and ready for serving. The camera focuses on the completed pizza, showcasing the even grid pattern and the appealing appearance of the cheese-topped slices. The person stands by, ready to serve the freshly cut pizza.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dfd333267d5a52251a49acaff3f885c86b6b7c9d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f41aa2fcf7407074301e512c6d00c267a3488a683be6649187b415426547c0b8 +size 1543171 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2aa22f8ddc87dafc0a0f2251e9e4bcba495051d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/console.log b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/console.log new file mode 100644 index 0000000000000000000000000000000000000000..64430ab9879aec0c906dd6ad15370ab197077700 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/console.log @@ -0,0 +1,22 @@ +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/config.yaml +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.json +[02-20 21:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:55:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:55:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:55:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:55:12|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/debug.log b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..87d8a5c856e491031a664080e5ac6f0dcc55f2d2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/debug.log @@ -0,0 +1,353 @@ +[02-20 21:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 21:41:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:41:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:41:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 21:41:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 21:41:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 21:41:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 21:41:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 21:41:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 21:41:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 21:41:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 21:41:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 21:41:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 21:41:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 21:41:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 21:41:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 21:41:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 21:41:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 21:41:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 21:41:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 21:41:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 21:41:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 21:41:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 21:41:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 21:41:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 21:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 21:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 21:41:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 21:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 21:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 21:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 21:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 21:41:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 21:41:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 21:41:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 21:41:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 21:41:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 21:41:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 21:41:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 21:41:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 21:41:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 21:41:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 21:41:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 21:41:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 21:41:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:41:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:41:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 21:41:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:41:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 21:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:41:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 21:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:41:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:41:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 21:41:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:41:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 21:41:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 21:41:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 21:41:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 21:41:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:41:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:41:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:41:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 21:41:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 21:41:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.26 s +[02-20 21:41:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:41:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:41:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 21:41:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/config.yaml +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 21:41:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the person continues to refine the white wooden piece, making final adjustments with the tool. The cat is now more engaged, playing with the cardboard tubes and exploring the workspace. The person remains focused on the task, occasionally interacting with the cat. The video ends with the person still working on the wooden piece, while the cat continues to play nearby.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.json +[02-20 21:41:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:41:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.mp4 +[02-20 21:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 21:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 21:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 21:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 21:55:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 21:55:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:55:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:55:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:55:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 21:55:12|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8feab60af280cde1177dfbc5a4607e1404fe53e6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a person sitting cross-legged on a tiled floor, holding a small white rectangular wooden piece and a tool. The person is dressed in an orange towel and a green shirt, with several bracelets on their wrist. Nearby, there are cardboard tubes and other wooden pieces scattered around. A Siamese cat with a blue collar appears from behind a window, watching the person curiously.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8bb9f23758b37bb618e09e69d953926c1f61da61 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6a26aeb3bd487df11db3ac9c83ea1d41aa235275967650d953c847ca333743c +size 1213541 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..e9312d84802e11f6fb9b9614b138b751ec472782 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"The person begins to measure the white wooden piece using a measuring tape, placing it carefully on the floor. The cat moves closer, sitting near the window and observing the activity. The person then sets the measuring tape aside and picks up a sanding block to smooth the edges of the wooden piece. The cat remains attentive, occasionally shifting its gaze between the person and the materials.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3402164158c7d0f30aff9df310d14a15dfc12815 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cce2b1e40c76cc1fc8479f87a63fa7e27af72b85f82cc9da649d6daa17d9a64 +size 1356331 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..00193c3cd3717860b11fb0d9cf6de07a1ce1e758 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Next, the person continues to sand the wooden piece, focusing on achieving a smooth finish. The cat moves around the workspace, sometimes getting closer to the person and other times retreating slightly. The person occasionally pauses to adjust the wooden piece or pick up another tool. The cat's curiosity is evident as it watches the person's every move.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f91ba0553efa429da9ea9a731788a7af09d8d4eb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6403af02a74e6eb697ec6b35c5fb82cc72a19c1250b885c29726a43a68d75e40 +size 1443825 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b869c4aa0264884d1b7b49848c32cf61bb3305c0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The person then picks up the sanded wooden piece and uses the tool to make precise cuts. The cat becomes more active, playing with one of the cardboard tubes and moving around the area. The person continues to work on the wooden piece, occasionally glancing at the cat, which adds a playful dynamic to the scene. The person adjusts the wooden piece and makes further modifications.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..646b8e806ab88831df30ef5ee1e2a2131cd79a33 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db089d43ad83e4b08250ad4172909ae1b93d8af3954fad6e8f81750e3299dcf9 +size 1936002 diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..73dd62a76054ac379dc829bd9a83634068b7ef26 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the person continues to refine the white wooden piece, making final adjustments with the tool. The cat is now more engaged, playing with the cardboard tubes and exploring the workspace. The person remains focused on the task, occasionally interacting with the cat. The video ends with the person still working on the wooden piece, while the cat continues to play nearby.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b182ba58d35d3613d55ca5ae8d6566f1a87aa413 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e87e8cd1e00bccd2755f13e360a42892f7326a66185393aabe96c77f32a8859 +size 1384086 diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccec28bcabe99450e044cc75b09f47b777aefd33 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/console.log b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/console.log new file mode 100644 index 0000000000000000000000000000000000000000..369ee33d0ae919dc77b8f08347cec4d30f137c89 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/console.log @@ -0,0 +1,22 @@ +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/config.yaml +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.json +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:37:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:37:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:37:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:37:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/debug.log b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..874e0d06152dd0aff0d63d192f4d65c66a6f882a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/debug.log @@ -0,0 +1,353 @@ +[02-20 07:21:48|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 07:21:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:21:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 07:21:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 07:21:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 07:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 07:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 07:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 07:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 07:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 07:21:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 07:21:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 07:21:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 07:21:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 07:21:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 07:21:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 07:22:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 07:22:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 07:22:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 07:22:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 07:22:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 07:22:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 07:22:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 07:22:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 07:22:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 07:22:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:22:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:22:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 07:22:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:22:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 07:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:22:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 07:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 07:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.05 s +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:22:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:22:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:22:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 07:22:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 07:22:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.23 s +[02-20 07:22:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:22:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:22:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/config.yaml +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 07:22:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the person finishes cleaning the sink area. They ensure all surfaces are wiped down and organized. The blue bowl remains in the left sink basin. The video concludes with the person making final adjustments to the countertop items, ensuring everything is tidy and clean.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.json +[02-20 07:22:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:22:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.mp4 +[02-20 07:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 07:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 07:22:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 07:22:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 07:37:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 07:37:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:37:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:37:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:37:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 07:37:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..1df7fda507e7789f3b2743cedf3f44c6afe703c7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a top-down view of a kitchen sink area. A person wearing red and yellow rubber gloves is cleaning the double stainless steel sink. The left sink has a bright blue plastic bowl, which the person picks up and washes under the running water from the chrome faucet. Various kitchen items, including cans and a white spray bottle, are visible on the white countertop against a blue tiled backsplash.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a86183db9e23239762ee3b64ba4bc288058a9ed7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:020c06690a28f42ca6bfc264f0c6450d6b723dd1d491e6d98c7616224a2f9eee +size 1159966 diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..f786e0a4a42969297768cfecef5f53ddd6aa5d47 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"After washing the blue bowl, the person places it on the counter. They continue cleaning the right sink, washing a red container and a small black object with a yellow sponge. The person then moves to the dishwasher, opens it, loads some items, and closes it. The blue bowl is then seen back in the left sink basin as the person prepares to clean the countertop area.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07841b4c9a65400c6458bd2e449ae17f70ffb99b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca760d59766334a89a29497356e1b3f0e098966ce514a052720d60adc2c5e9f1 +size 2279282 diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..78a90b3452aa9d8e90302267734676a7fb85da6c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person begins cleaning the countertop around the sink, moving containers such as a green-lidded one and a clear pitcher to clean underneath them. The blue bowl remains in the left sink. The person then sprays a cleaning solution onto the sink and wipes it down thoroughly with a cloth. The camera briefly pans to show a dining area in the background.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c443987ae0ca724f27bca5a34cbe5ccd0380742c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f40d11fb95ee3fa43c2818c454ffb7f253a11437bd14b237526a72ab4f21367 +size 2369102 diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..3668556155a1e25cffc8a0cb0eba595f11fd7ba5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The person continues to clean the sink area, focusing on the right side. They spray a cleaning solution from a bottle labeled 'FAIR' onto the sink and wipe it down meticulously. The blue bowl is still in the left sink basin. The person then organizes the countertop, placing lids and containers neatly next to the sink.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f822918b39a0d240c7733de18983f46306f3acfb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff726c9015c3ca281499eb59136251232ff62dd47736591a41a26ae273ddf2c8 +size 1879700 diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..be24b369aa22c3357768e786a76b60fe344f9fb1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the person finishes cleaning the sink area. They ensure all surfaces are wiped down and organized. The blue bowl remains in the left sink basin. The video concludes with the person making final adjustments to the countertop items, ensuring everything is tidy and clean.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea84668295dd74ef0c73b833cb8cc0e2dde6fb98 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:787ffb7d63c82b4622a73af2ba535e560944218a1ca8cdc7a614111e4e155dbd +size 1492129 diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f42b7564a3445bc491dbe13716f34fa682f44e3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/console.log b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/console.log new file mode 100644 index 0000000000000000000000000000000000000000..0ca7bf5b8d72f0f1a0c11bf21ae05bbff8343d16 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/console.log @@ -0,0 +1,22 @@ +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:14:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:14:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/config.yaml +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.json +[02-20 17:15:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:28:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:28:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:28:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:28:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/debug.log b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..22ddb06f33036a0076ddfa92ff24ee199bb9f5ba --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/debug.log @@ -0,0 +1,353 @@ +[02-20 17:14:29|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 17:14:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:14:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:14:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 17:14:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 17:14:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 17:14:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 17:14:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 17:14:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 17:14:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 17:14:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 17:14:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 17:14:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 17:14:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 17:14:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 17:14:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 17:14:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 17:14:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 17:14:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 17:14:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 17:14:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 17:14:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 17:14:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 17:14:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 17:14:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 17:14:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:14:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:14:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 17:14:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:14:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:14:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:14:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 17:14:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:14:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 17:14:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:14:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:14:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 17:14:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:14:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:14:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:14:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:15:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 17:15:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 17:15:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.72 s +[02-20 17:15:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:15:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:15:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 17:15:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/config.yaml +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 17:15:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final segment, the camera focuses on the person's hands as they continue to tidy up the remaining books on the floor. The video concludes with the person still engaged in organizing the books, with several titles visible on the shelf, including 'Conquest' and 'Desperate Crusade'. The overall atmosphere remains calm and focused on the task of tidying up." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.json +[02-20 17:15:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.mp4 +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 17:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 17:15:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 17:15:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 17:28:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 17:28:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:28:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:28:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:28:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 17:28:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3af932aa13f221a388af3ce0c1a64cd4eb881e0d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a close-up shot of an open book with cream-colored pages and black text, titled '190 • Great Divide', resting on the viewer's lap. The setting is a cozy living room with a glass coffee table, decorative items, and patterned rugs. The person appears to be relaxing and reading in a comfortable chair.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e04bb80d88d93888e4f553696c1bb268c62d5145 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b9b907519ced9b1560b597d67400e0a00f4dd22af8e9e2e72485180d59bcf9f +size 889183 diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5b217100072b1d61e29c30ca2aa89f9862157da5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the camera suddenly moves erratically, shifting focus away from the book and revealing a large pile of books scattered across the floor near a white shelf. The person's hand is briefly visible, indicating they are about to start cleaning up the mess.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7f26012833b9d43d416a0de9b97a5b1725e81e91 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43aa85dd2398396e6869af1eaf9972a0c5f7bdb0adbf7ecbe09340b33f01f2c5 +size 1300988 diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..9a4353fc2655a2fc8948f7dba085a338136e74fd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The next part of the video shows the person actively picking up the books from the floor and placing them back onto the white shelf. Various book titles become visible, including 'The Perfect Assassin' and 'American Heritage'. The camera follows the person's hands as they sort through the books, occasionally showing the covers of different titles.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f7cead022074d07be3a3e8c4d7215762378194f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8f015703aa9d60c78a6db28bf330749fd9bd591372157ec086ef2a167b2ea47 +size 1438076 diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..89b01727c4d82cf8ed1b61062d132f2819b32ee1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The person continues to organize the books on the shelf, carefully arranging them in a neat order. The camera captures the process from different angles, highlighting the variety of book covers and titles being placed back onto the shelf. The living room setting remains consistent with the glass coffee table and patterned rugs in the background.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f870c7c12a444d51b750c298b7b589dfd55152a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d38e93dc39499323a3f061bccda5f44e2716c1a455f11f3f56f2e844d08b980 +size 1763819 diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..1b0ac9739a7c36acc66c28b923c8e9cedfc77a26 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the camera focuses on the person's hands as they continue to tidy up the remaining books on the floor. The video concludes with the person still engaged in organizing the books, with several titles visible on the shelf, including 'Conquest' and 'Desperate Crusade'. The overall atmosphere remains calm and focused on the task of tidying up.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57c8f85f3a603215285505888bbd15370bad2171 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5e2e1af60f864378885c344806a52235ecff667990473448d73b7f15311f140 +size 842553 diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0164328b2fe96f48b045e40f57cd2500db07e55 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/console.log b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/console.log new file mode 100644 index 0000000000000000000000000000000000000000..3da3f324faad70bfd7abab2b27086e63fc8d5880 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/console.log @@ -0,0 +1,22 @@ +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/config.yaml +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.json +[02-19 22:55:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:09:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:09:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:09:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:09:32|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/debug.log b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..caf8e5a68b0c11beb9c4f2ccfd0cdc17ea3f7758 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/debug.log @@ -0,0 +1,353 @@ +[02-19 22:54:19|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 22:54:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:54:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 22:54:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 22:54:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 22:54:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 22:54:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 22:54:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 22:54:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 22:54:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 22:54:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 22:54:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 22:54:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 22:54:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 22:54:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 22:54:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 22:54:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 22:54:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 22:54:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 22:54:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 22:54:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 22:54:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 22:54:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 22:54:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 22:54:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 22:54:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 22:54:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 22:54:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 22:54:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 22:54:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 22:54:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 22:54:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 22:54:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 22:54:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 22:54:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 22:54:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 22:54:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 22:54:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 22:54:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 22:54:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 22:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 22:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 22:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:54:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 22:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 22:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 22:54:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 22:54:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:54:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-19 22:54:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 22:54:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 22:54:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 22:54:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 22:54:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 22:54:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 22:54:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:55:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 22:55:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 22:55:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.40 s +[02-19 22:55:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:55:06|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:55:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/config.yaml +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 22:55:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the person completes the painting process, stepping back to inspect their work. The camera captures the fully painted handrail, showcasing the smooth finish and the enhanced appearance of the wood. The person appears satisfied with the results, having transformed the handrail through their diligent efforts.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.json +[02-19 22:55:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:55:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.mp4 +[02-19 22:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 22:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 22:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 22:55:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:09:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:09:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:09:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:09:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:09:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:09:32|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..1fba87ac4b2be932edbf0286edce99bee845560a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with the person sitting on the stone steps of a spiral staircase, sanding the light brown wooden handrail. They use their hands to smooth the surface, moving along the railing to ensure even sanding. The camera focuses on their actions, capturing the texture of the wood and the details of their movements.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6c38c46a3905e6685fcb35980c70503be04aacad --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90db4d8a304cfd66f020942fcc4dff2db254b235b4c3dc8692d53410f32e6005 +size 1562355 diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..909b84a69f217be42cac9d8e6d3229579ae154f9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the person continues, they transition from sanding to painting the handrail. They pick up a paintbrush and begin applying paint in smooth, even strokes. The camera follows their hands, showing the gradual transformation of the wood as it is coated with paint. The surrounding garden area with green plants becomes more visible as the person moves around the staircase.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68299f40f63e76fc1eab9a40f9650fd58b965887 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:264ee26c859ac514fb9d515c8ec7604d33cbbebb29fb496650cf02fc6b6e120a +size 1949035 diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..fb8d561de2109eec4a4676f476875656d126ac71 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person meticulously paints the entire length of the handrail, paying attention to the edges and corners. They adjust their position on the steps to reach different sections, ensuring complete coverage. The camera captures the progress, showing the contrast between the painted and unpainted areas of the wood.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa0ad7ed12e0cc4b96b8ca537521c59e14fae0e7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94f65db49d9d8671eae1012e6c9936802e235b09ac119cc1574463ce55d75f88 +size 1462384 diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9f74bfcb028fc7f0294b8a46ff1575048bf9434c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Moving further along the staircase, the person continues painting the handrail, focusing on the lower sections. The camera angle shifts to provide a broader view of the staircase and the garden below. The person's careful application of paint is evident as they work their way down the railing.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..193a43b9c15ba7e098528b18355aacefdb4cfab5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2beb97678dc309b5b053c5581df07fb674c576c8c49881a301230696edca6130 +size 951601 diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..24fb9f781f4cc53968109d4c3de98b0aa7716cb8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person completes the painting process, stepping back to inspect their work. The camera captures the fully painted handrail, showcasing the smooth finish and the enhanced appearance of the wood. The person appears satisfied with the results, having transformed the handrail through their diligent efforts.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..887e99c85858bfadbf9e54492f323c8fdba1b717 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bc8800362c47c1a640b705d04b0accfdd7939d89efca7e2e38ccb5613712b28 +size 840208 diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..347f8e4d5be75382d24e093bfe68299623a26ade --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/console.log b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/console.log new file mode 100644 index 0000000000000000000000000000000000000000..dd59b1d0d39a287e7e22fc7fa9a6ba02661b988b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/console.log @@ -0,0 +1,22 @@ +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 20:29:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 20:29:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/config.yaml +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.json +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 20:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 20:43:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 20:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 20:43:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/debug.log b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3c8a8a72eb717d32407c8d2873c59c7fcabe9780 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/debug.log @@ -0,0 +1,353 @@ +[02-20 20:29:09|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 20:29:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 20:29:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 20:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 20:29:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 20:29:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 20:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 20:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 20:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 20:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 20:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 20:29:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 20:29:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 20:29:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 20:29:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 20:29:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 20:29:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 20:29:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 20:29:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 20:29:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 20:29:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 20:29:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 20:29:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 20:29:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 20:29:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 20:29:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 20:29:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 20:29:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 20:29:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 20:29:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 20:29:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 20:29:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 20:29:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 20:29:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 20:29:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 20:29:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 20:29:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 20:29:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 20:29:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 20:29:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 20:29:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 20:29:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 20:29:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 20:29:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 20:29:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 20:29:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 20:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 20:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 20:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 20:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 20:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 20:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 20:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:29:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 20:29:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 20:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 20:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.04 s +[02-20 20:29:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 20:29:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:29:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/config.yaml +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 20:29:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the person completes the spinning process, with a significant length of white yarn now formed. The orange fabric on the lap is almost entirely covered with the accumulated yarn. The individual makes final adjustments to the yarn, ensuring it is neatly wound and properly tensioned. The background, featuring the colorful mat, wooden chair, and basket of cotton, remains unchanged. The camera captures the completed yarn and the person's hands as they finish the task, providing a clear view of the finished product and the spinning tool." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.json +[02-20 20:29:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 20:29:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.mp4 +[02-20 20:29:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 20:29:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 20:29:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 20:29:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 20:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 20:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 20:43:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 20:43:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 20:43:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 20:43:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..db3b5dbbf43fe3e230c0eeb1177ad4632ad9d988 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of a person's hands as they begin to spin cotton using a charkha. The individual is wearing a green sari with red paisley patterns and has an orange fabric draped over their lap. They hold a white cotton ball in one hand and operate the silver-colored spinning tool with the other. The initial stage shows the cotton loosely wound around the spindle, with the person starting to twist it into yarn. The background features a colorful circular mat, a wooden chair, and a basket containing additional cotton balls. The camera focuses on the hands and the spinning tool, capturing the beginning of the yarn-making process.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6987a32c94d684516bc806b468717f5bc7057ccf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88a9579b96863997079711e56b0652e85dfb727893dcb37cd816e4a145e2651f +size 1341285 diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..35ae0019f0e5132d9fa6ba0831fef67ba33a6c3b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the spinning progresses, the person continues to pull and twist the cotton, forming a continuous thin white yarn. The orange fabric on the lap serves as a surface for the spinning tool and the accumulating yarn. The individual occasionally adjusts the tension of the yarn and repositions the cotton ball. The colorful mat and wooden chair remain visible in the background, providing context to the setting. The camera maintains its focus on the hands and the spinning process, capturing the rhythmic motion of the charkha as the yarn begins to take shape.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70ed8b473c64357beb1c701cdf5a3e5293c82fce --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:589f59ace605fa7db2f640256336525fc3d0db25f8673260246ad1df2706be46 +size 1638044 diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..76aeec2d77bc7ed5e03b0d2ee85d791a9a72599e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person continues to spin the cotton, with the yarn becoming longer and more defined. Their hands move with precision, ensuring the yarn is evenly twisted. The orange fabric on the lap now has a noticeable amount of white yarn laid out on it. The background elements, including the colorful mat, wooden chair, and basket of cotton, remain consistent. The camera angle shifts slightly at times, offering different perspectives of the spinning process while keeping the focus on the hands and the emerging yarn.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a201396f099ebe17393959de67e0f9a8400d003e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7e8db83d148be1aa61fa33262f966b2d11985c92aa25c9b0a951ed1a6d53d41 +size 2018748 diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4c9b6a81248e8b248780bd56296f063feb76e2cf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In this segment, the person further develops the yarn, pulling and twisting the cotton with practiced ease. The orange fabric on the lap is increasingly covered with the white yarn, indicating progress in the spinning task. The individual occasionally pauses to adjust the cotton ball or the tension of the yarn. The colorful mat, wooden chair, and basket of cotton continue to be visible in the background, maintaining the setting's consistency. The camera remains focused on the hands and the spinning tool, capturing the detailed movements involved in the craft.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9cb2656cb43bcacf2e8411ab713c2bdfa911eb1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2765d9c499f5a56fd3c2313e458f59b13746107fc3755a97d931c6a099be5ff6 +size 1359116 diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c1ce7f3f10e6d9d4cee8cdd826e48269e25a35bd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person completes the spinning process, with a significant length of white yarn now formed. The orange fabric on the lap is almost entirely covered with the accumulated yarn. The individual makes final adjustments to the yarn, ensuring it is neatly wound and properly tensioned. The background, featuring the colorful mat, wooden chair, and basket of cotton, remains unchanged. The camera captures the completed yarn and the person's hands as they finish the task, providing a clear view of the finished product and the spinning tool.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d5d26fbceeb3cb0590a4ef5a2c3dbaae84122f38 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:854ab87e718b7ffc7e300b1e70458ed6912e0d56733be5e9ca01f03412d1dd69 +size 1371119 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bb6c9e9a8b91cd4187c51131806b6b925a9d268 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/console.log b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ee83f58521f3892714a9703f514888a1040e4b06 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/console.log @@ -0,0 +1,22 @@ +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:53:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:53:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:54:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:54:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/config.yaml +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.json +[02-20 06:54:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:07:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:08:00|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/debug.log b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a9bbe7d0a7d5093b567f998aef4f6403e19cd0a0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/debug.log @@ -0,0 +1,353 @@ +[02-20 06:53:45|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 06:53:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:53:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:53:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 06:53:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 06:53:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 06:53:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 06:53:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 06:53:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 06:53:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 06:53:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 06:53:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 06:53:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 06:53:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 06:53:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 06:53:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 06:53:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 06:53:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 06:53:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 06:53:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 06:53:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 06:53:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 06:53:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 06:53:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 06:53:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 06:53:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 06:53:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 06:53:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 06:53:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 06:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:53:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:53:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 06:53:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:53:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 06:53:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:53:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:54:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 06:54:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:54:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 06:54:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 06:54:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 06:54:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 06:54:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:54:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 06:54:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 06:54:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 06:54:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:54:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:54:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 06:54:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 06:54:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.23 s +[02-20 06:54:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:54:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:54:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 06:54:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/config.yaml +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 06:54:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the person finishes loading the washing machine with all the clothes from the pile and prepares to close the lid, completing the laundry task.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.json +[02-20 06:54:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.mp4 +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 06:54:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 06:54:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 06:54:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 07:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 07:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:07:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:07:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 07:08:00|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2f823412246132a74706055929c9bbd35e357e65 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins in a kitchen where a person is cooking a dish in a pan. They use a blue plastic container with a teal lid to store some of the prepared food. The person adds spices to the pan and then picks up the container, moving it out of the frame.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5350efa179b8434c776d6ec72bed42a17174b422 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f065ef19fd1fb27e99eea665303ec8a72d44a58e3f9eccbd4b29200456e718 +size 1078814 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..c8f029340ad17404522355ac2de4c05f5e24cdb7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"After leaving the kitchen, the person walks through a hallway and enters a bedroom. In the bedroom, they approach a washing machine, open its lid, and begin placing clothes inside from a pile on the floor.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c761c0b486e6dfa66fc3e537ed6a8f766516003 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee60d8cbe13d36b9f616b34f127bb82636e5d6deee4492447b30c5642715cbcc +size 1043225 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..18505db0c48fd9c282f47efab3ef8ce99720afaf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person continues to load the washing machine with various clothes, including shirts and other garments, sorting through the pile and placing each item inside the machine.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3d15627f705a6cdb5675e4171d86c922418a7997 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e6447f5c930819a4ade2f474e2b8d73343ae510d39359a87688533aeef97348 +size 791689 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a8b4fa90d82d208d4824fed9447ba540981c46bf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the person loads more clothes into the washing machine, they organize the garments carefully, ensuring everything fits inside. The camera focuses on their hands as they handle the clothes.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e9caa7af85e0d043254a3048d07d217c4899b18 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f48f18c35eb2fc35451e3c0dc6ccbf5a7851ac08fef14c894e2cf3388ae04fef +size 1550643 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..74b314cbc6c02bcb735774cfba3774408855e33a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person finishes loading the washing machine with all the clothes from the pile and prepares to close the lid, completing the laundry task.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4bae0ffd20fc8bc1cab76ef1cc6629330ae386a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6a07fe15ec4dedc155e359b70e7536be7a597d6fd8f934cf0d25ef786d9244 +size 1275000 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22ee55f9e3686441b50e7f5a2ce00e12defcf13c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/console.log b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/console.log new file mode 100644 index 0000000000000000000000000000000000000000..377583c218d1ea01753dfd1b08bafd30558fb435 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/console.log @@ -0,0 +1,22 @@ +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/config.yaml +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.json +[02-20 08:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:55:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:55:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/debug.log b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..57aad815020e918bb2d8dff9c44c0d3e416f92bd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/debug.log @@ -0,0 +1,353 @@ +[02-20 08:41:45|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 08:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 08:41:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 08:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 08:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 08:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 08:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 08:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 08:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 08:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 08:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 08:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 08:41:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 08:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 08:41:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 08:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 08:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 08:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 08:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 08:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 08:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 08:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 08:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 08:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 08:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 08:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 08:41:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 08:41:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 08:41:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 08:41:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 08:41:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 08:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 08:41:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 08:41:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 08:41:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:41:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:41:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 08:41:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:41:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 08:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:41:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 08:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:41:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 08:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 08:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 08:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 08:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 08:42:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:42:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:42:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:42:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:42:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 08:42:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 08:42:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.37 s +[02-20 08:42:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:42:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:42:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 08:42:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/config.yaml +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:42:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the man applies glue to the newly cut wooden pieces and fits them into the frame, continuing the assembly process. He then uses a pneumatic nail gun to secure the pieces, completing the frame. The video concludes with a close-up of the finished wooden frame, showcasing the precision and care taken throughout the project.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.json +[02-20 08:42:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:42:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.mp4 +[02-20 08:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 08:42:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 08:42:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 08:42:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 08:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 08:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:55:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:55:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:55:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 08:55:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..7c89849c976e5dd2d26e6f946d25b9f23d7b3f33 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a man in a white t-shirt using an orange and silver RIDGID circular saw with a blue light to cut a piece of wood in a cluttered workshop. The saw is prominently visible with its brand name 'RIDGID' clearly shown. After cutting, he sets the saw down and begins assembling a wooden frame using smaller wooden pieces, clamps, and a tape measure, focusing on precise measurements and alignment.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11eac9b59be899c268329d625050ac8e0c2e7652 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3646bf544f820c783148fcf490452f743c1cad5b54ba8c40b94e8c06de542ce +size 1037152 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..33886081d1ed414fbc31e5e07b001277bf1f2ab9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the previous scene, the man applies glue to one of the wooden pieces and carefully fits it into the frame, ensuring a snug fit. He then moves around the workshop, gathering additional tools such as a pneumatic nail gun from a nearby shelf. The environment is filled with various tools and materials, emphasizing the busy nature of the workspace.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51b24d6cba93e01573dd8c35f79311e105ea164e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e75f8f617dc558c80d7535b3eee526bbdbe0b20bfc75ba1750e17b4bc57d8685 +size 1314398 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..b81950b539265e96c71528e5ca0fc152d0b8cde6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The man proceeds to apply glue to another wooden piece and fits it into the frame, making sure all components are securely in place. He then returns to the RIDGID circular saw to cut another piece of wood, demonstrating the tool's reuse after several steps involving other materials and tools. The focus remains on the detailed assembly process within the cluttered workshop.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b094257e6a2f4bbdb5240025306929b18e8e71b5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0d6978bb6c648cae8c21ec6e03af0be805f9dfe0be2c44a52d2b9960aac1c74 +size 1313310 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..1c1bc75d332017737de95180029a3fb39ab6babe --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the next segment, the man uses the circular saw to make precise cuts on additional pieces of wood, preparing them for the frame. He ensures each cut is accurate before moving on to the next step. The workshop remains filled with tools and materials, highlighting the ongoing construction activity.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ff2f104efcac4c4cda639352d4543ebef2d1258 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f82914234d0d22dbc61c711e8f8b3f407f2e994adb7a116db3817966b80369ca +size 1264548 diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..eaee9304d1a729561c29224f3a8af56a0eef66ae --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the man applies glue to the newly cut wooden pieces and fits them into the frame, continuing the assembly process. He then uses a pneumatic nail gun to secure the pieces, completing the frame. The video concludes with a close-up of the finished wooden frame, showcasing the precision and care taken throughout the project.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..692a82de068d7316b322be740a93ff6dd852de2d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dd52b2693ce325a2ea6fb3ce598983016063a976dc2878843e0e90a62ca6c54 +size 1201946 diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc727ce514d731ef451d5655d6656890e277860e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/console.log b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/console.log new file mode 100644 index 0000000000000000000000000000000000000000..bd25219bec9ac31f9b0976d49918dce1d4da22ac --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/console.log @@ -0,0 +1,22 @@ +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:20:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:20:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/config.yaml +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.json +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:34:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:34:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:34:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:34:11|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/debug.log b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..314da4db0817e91d869d66c670b7ec0284f12fe6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/debug.log @@ -0,0 +1,353 @@ +[02-20 17:20:01|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 17:20:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:20:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:20:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 17:20:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 17:20:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 17:20:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 17:20:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 17:20:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 17:20:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 17:20:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 17:20:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 17:20:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 17:20:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 17:20:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 17:20:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 17:20:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 17:20:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 17:20:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 17:20:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 17:20:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 17:20:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 17:20:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 17:20:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 17:20:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 17:20:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 17:20:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 17:20:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 17:20:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 17:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 17:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 17:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 17:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 17:20:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 17:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 17:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 17:20:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:20:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:20:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 17:20:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:20:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 17:20:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:20:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:20:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 17:20:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:20:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:20:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 17:20:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 17:20:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.28 s +[02-20 17:20:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:20:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:20:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/config.yaml +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 17:20:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The video concludes with the man finalizing the arrangement of the cards on the table, ready to start the game. The camera provides a final overview of the scene, showing the 'Sorry!' box, the arranged cards, and the comfortable living room setting with natural light streaming in." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.json +[02-20 17:20:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:20:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.mp4 +[02-20 17:20:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 17:20:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 17:20:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 17:20:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 17:34:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 17:34:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:34:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:34:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:34:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 17:34:11|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e61832cde5137e2acc8d70616b5f9b026e6111fd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of the man holding a deck of red, white, and blue playing cards, examining them closely. He is seated on a couch in a bright living room with large windows showing greenery outside. A 'Sorry!' game box sits on the glass coffee table in front of him, and a potted plant is visible near the TV stand.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73587393d6b44bb7ae442fe0da8ff240dcb25b8b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a8e2978ec6e3a67c63fd4b4852a6b8a7e23c95aa242be5c0dbaa5c03a0dd170 +size 888993 diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..67b1916a4e7a941019dac8f760c05f9e7885559d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the man begins to shuffle the cards, with the camera capturing his movements from different angles. He places some cards on the table while continuing to hold others. The 'Sorry!' box remains a focal point on the table, and the background includes the window, plant, and TV stand with various items.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f6fa7506e672e3795ee7f3458e1b70816a815c22 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdf60af89b1e810445a5ed1d3d1c7b0986c4fa26ae20a7098577686bb83e9186 +size 1031712 diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..f8f320420bc1de0227682dcf81d030bead4b78cc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The man continues to arrange the cards on the table, occasionally looking at the 'Sorry!' box. The camera moves around, providing side views and overhead shots of his actions. The natural light from the window enhances the visibility of the cards and the game box, maintaining a consistent focus on these objects.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf667cf71aeb08c81bf3626216a2f7006dd341dc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ebc66f641a1f549b0a0cc62b76fe821138f2dd909ee11682d28831a6efaaf3e +size 1487928 diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..193f9ab015e90dc694b8c94410f771665c66ac03 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the next segment, the man is seen placing the cards into a specific order on the table, preparing for the game. The camera captures his focused expression and the detailed movements of his hands. The 'Sorry!' box remains stationary, and the background elements, including the window and plant, are consistently visible.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dae6de01a9be79ac55274af1b761cd62494a5596 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9420241b810661ae7280ea15859c2e71c39bc8c163dd16fe6caeda838966e39 +size 1336608 diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..a721ac8ed512c309bab7be31add6a7afc3055c8f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with the man finalizing the arrangement of the cards on the table, ready to start the game. The camera provides a final overview of the scene, showing the 'Sorry!' box, the arranged cards, and the comfortable living room setting with natural light streaming in.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1ff331b693f70313da6b85ecd24bf763dae2b7d9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46af9cc928d9a7bb89bb8e080d267d94a34050cd79a72fb49dd295957636c945 +size 1136671 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ad69c21aa29c271a07cbff04ddf06b62b22a801 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/console.log b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/console.log new file mode 100644 index 0000000000000000000000000000000000000000..bcaa191df780f16206275ee94c1c8abfce24d399 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/console.log @@ -0,0 +1,22 @@ +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/config.yaml +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.json +[02-20 03:56:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:09:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:09:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:09:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:09:22|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/debug.log b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d8c0518ee7d03787ff259f072d9a15bda1636f06 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/debug.log @@ -0,0 +1,352 @@ +[02-20 03:55:17|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 03:55:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:55:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:55:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 03:55:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 03:55:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 03:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 03:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 03:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 03:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 03:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 03:55:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 03:55:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 03:55:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 03:55:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 03:55:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 03:55:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 03:55:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 03:55:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 03:55:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 03:55:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 03:55:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 03:55:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 03:55:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 03:55:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 03:55:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 03:55:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 03:55:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 03:55:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 03:55:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 03:55:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 03:55:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 03:55:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 03:55:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 03:55:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 03:55:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:55:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:55:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 03:55:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:55:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 03:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:55:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 03:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:55:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:55:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:55:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:55:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 03:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 03:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.24 s +[02-20 03:55:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:56:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:56:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 03:56:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/config.yaml +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 03:56:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a close-up shot of a person's hand using a screwdriver to remove a silver metal hinge from a white cabinet door. The door is rectangular and has a smooth texture. The person carefully unscrews the hinge and places the screwdriver on the carpeted floor. Next, the person moves around the room, walking past a desk with a computer and other items, then heading towards a staircase and a storage area filled with boxes and tools. The person continues to gather materials and tools from various parts of the house, including a kitchen area. Finally, the person returns to the original room, picks up the white cabinet door, and carries it towards a cardboard box near a window." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.json +[02-20 03:56:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 03:56:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 03:56:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 03:56:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 04:09:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 04:09:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:09:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:09:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:09:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 04:09:22|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ad8e6ad59b1a9c231297826e8c07ff04fd7a584b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a close-up shot of a person's hand using a screwdriver to remove a silver metal hinge from a white cabinet door. The door is rectangular and has a smooth texture. The person carefully unscrews the hinge and places the screwdriver on the carpeted floor. Next, the person moves around the room, walking past a desk with a computer and other items, then heading towards a staircase and a storage area filled with boxes and tools. The person continues to gather materials and tools from various parts of the house, including a kitchen area. Finally, the person returns to the original room, picks up the white cabinet door, and carries it towards a cardboard box near a window.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41e2b7a30a860f5811684a5ab48446f72516f25b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:736e25e6b0919a71c775255207d72d1d832488c0c53c1c518858ccae6819b38f +size 1870629 diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e6a658a19610e5628c183bd23a64fa49eb90c3f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/console.log b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/console.log new file mode 100644 index 0000000000000000000000000000000000000000..1d2b11674912b6a2899ebec0947c8bf92567678f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/console.log @@ -0,0 +1,22 @@ +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/config.yaml +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.json +[02-19 20:48:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:01:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:01:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/debug.log b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cc3ff3a361915a4c08b80ee0b0974c5fd775c14a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/debug.log @@ -0,0 +1,353 @@ +[02-19 20:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 20:47:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 20:47:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 20:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 20:47:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 20:47:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 20:47:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 20:47:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 20:47:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 20:47:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 20:47:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 20:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 20:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 20:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 20:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 20:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 20:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 20:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 20:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 20:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 20:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 20:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 20:47:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 20:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 20:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 20:47:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:47:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 20:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:47:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 20:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 20:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-19 20:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 20:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 20:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 20:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.67 s +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.68 s +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.68 s +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:47:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 20:47:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 20:47:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.13 s +[02-19 20:47:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:48:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/config.yaml +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:48:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='The video concludes with the shopper carrying the red shopping basket, now containing the maroon knitted hat, as they approach the exit of the store. The camera captures the final view of the basket and the surrounding store environment, with the shopper preparing to leave after their shopping experience.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.json +[02-19 20:48:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:48:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.mp4 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 20:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 20:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 21:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 21:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:01:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:01:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 21:01:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..4bd714625b79d4d71f72955c7af6334030e2fa8c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a first-person view of a red shopping basket being carried through a clothing store. The basket contains a white fluffy object, and the camera focuses on a display of black belts and blue ties on a marble table, with mannequins dressed in stylish outfits in the background. The shopper walks past this display, entering a section with hanging clothes near large windows.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa5400dad3c3b295defc9ebaba8dde8a836a6ef5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cd700cab7458a04eaaa498da1d48763449b47b87540ef6dee5da835cf6b7dac +size 1699850 diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..86c0ebe0b4d5845761764b496211847fc4b0f0ab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the shopper moves through the store, the camera captures a close-up of the red shopping basket, showing its handle and contents. The shopper passes by a mirror and racks filled with various shirts and jackets, with another person visible in the background organizing items. The camera then shifts to a section with jeans and other garments on slatted walls, focusing on a gray long-sleeve shirt and a pair of jeans.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11c612bf6baffec0b74c5d953c0b575b6d96fb47 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea3dae4b1b6c3463887071bae26e3a43bd0a4da1118508dea9f92d0e5af88d01 +size 2438517 diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..8d4f2404860e6ec35e61947a3ab6b9c93059121a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The shopper continues to browse through the store, approaching a display of a maroon knitted hat on a mannequin. The camera zooms in on the hat as the shopper picks it up and inspects it closely. The shopper then moves towards a section with a sign advertising a 'Flash Sale' with 25% off, passing by a mannequin dressed in a maroon outfit.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92b9e2fe515dbfd67027d1f94f6f3ed106fd0345 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:579cb1494aa3bb6510ca6a39256909ed2a654e6e5f4bd0868f9a3d287811bc95 +size 1789851 diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..946dbd485cb4976ed8d5d41c97b464c62294a9d3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The shopper examines the maroon knitted hat more closely, turning it around to check its details. The camera then pans back to the red shopping basket, now containing the hat, as the shopper continues to walk through the store. The camera captures a brief view of the initial display of belts and ties before the shopper moves towards the exit.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..256fa43c2ad88f54e4fcf10373460b3ef0159a4a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa4631ccea547b44e5ecc85566cd80b1dd81e075d4e939990dea4d76adb4c691 +size 1782337 diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..930cccf09bc2ec74c5c6337fe9bdc7b2e9674107 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with the shopper carrying the red shopping basket, now containing the maroon knitted hat, as they approach the exit of the store. The camera captures the final view of the basket and the surrounding store environment, with the shopper preparing to leave after their shopping experience.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..438be9701fe27c25c7c09735cf84570f4ff98e68 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41e42cda07f926e5c4e56e24c64da981301206d04ff2fba668845934afeb8843 +size 2295269 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..634d42b3b915e0f264706bb445b52175927585c6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/console.log b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/console.log new file mode 100644 index 0000000000000000000000000000000000000000..8bf2bb81a111005498fb6711aceb245a9b18eac0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/console.log @@ -0,0 +1,22 @@ +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/config.yaml +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.json +[02-20 16:09:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:22:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:22:31|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/debug.log b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7e62eff47c5ff68367ec96d25e7755c6aab72ff3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/debug.log @@ -0,0 +1,353 @@ +[02-20 16:08:23|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 16:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:08:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 16:08:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 16:08:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 16:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 16:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 16:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 16:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 16:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 16:08:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 16:08:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 16:08:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 16:08:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 16:08:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 16:08:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 16:08:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 16:08:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 16:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 16:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 16:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 16:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 16:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 16:08:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 16:08:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 16:08:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 16:08:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 16:08:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 16:08:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 16:08:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 16:08:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 16:08:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 16:08:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 16:08:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 16:08:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:08:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:08:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 16:08:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:08:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 16:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:08:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 16:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 16:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:08:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 16:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 16:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.69 s +[02-20 16:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:09:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:09:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 16:09:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/config.yaml +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 16:09:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the person holds up a pair of black pants, preparing to pack them. The camera captures the details of the pants as the person folds them carefully. The person then places the pants into the suitcase, continuing to organize the contents. The video concludes with the person standing next to the dresser, having sorted through most of the clothes in the drawer.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.json +[02-20 16:09:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.mp4 +[02-20 16:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 16:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 16:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 16:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 16:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 16:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:22:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:22:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:22:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 16:22:31|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ecee88af60f627ae8f4ab961dd1cf45639a3157f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a first-person view of a cluttered bedroom. The camera focuses on an olive green dresser with gold handles. The person, dressed in a red and white striped shirt, opens a drawer and begins to organize the clothes inside. A small dog is visible in the background, moving around the room. The camera pans to show other areas of the bedroom, including a kitchenette and a hallway, before returning to the dresser.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63145a48c99ef2ce1f83eca03d5fa0ef31031c65 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:680fdd842852593ca5242b5a81e7a833745fc28209196e864361e626880fad49 +size 1836862 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..edb8d121cb5878f89a01034506cf45155695e709 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the previous scene, the person continues to sort through the clothes in the drawer of the olive green dresser. The dog is seen lying on the floor near a guitar case and a trash can. The person picks up various garments, examining and folding them. The camera captures the details of the room, including a bookshelf filled with books and a lamp on a side table.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e448cad8581e16b974d4c575fa3702143237f6ac --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:907846b561c9bb6831efe182be4e66109efffce05a5975c01017178d6a740233 +size 2297551 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..ae422a14ffce0f23bac24dde1aafebef08b7fb2e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person then closes the drawer and moves towards the suitcase on the floor. The camera provides a close-up view of the suitcase as the person begins to pack clothes into it. The person folds and arranges the clothes carefully, ensuring they fit neatly. The camera captures the details of the suitcase's contents, showing the organized arrangement of the clothes.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4b624b01808027e49acf5c24850a89e3f857c5e9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a219c13fee97a103931b2602dde0628ba0e1c94a59efe307ae7019808e89bc96 +size 1829669 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9ec7ee5f9892fe3028547093feac0be0bebbc32b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Next, the person returns to the dresser and opens another drawer. The camera shows a close-up of the drawer as the person continues to sort through more clothes. The dog is seen lying on the floor near the dresser, watching the person. The person picks up various garments, examining and folding them before placing them back in the drawer.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..83f45663fc2acf0f73fb18fc5f62e483b35b55b8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9b7332d89f7281b0ee4615cc6fcc27a138ddcc121c41de7da010c7df0966536 +size 2021422 diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..350f547a5c659b99f13671f45e82fdc0a7312330 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person holds up a pair of black pants, preparing to pack them. The camera captures the details of the pants as the person folds them carefully. The person then places the pants into the suitcase, continuing to organize the contents. The video concludes with the person standing next to the dresser, having sorted through most of the clothes in the drawer.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..182f4e57a209fce709c40425c3998a80226e8086 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d28f928eb4291e277b747f46de353d5fbfacb1c7284ba14846515aeb5c9f3f56 +size 1894605 diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ffe9c1e0c5e77011a10bae020d9b317a79ea194 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/console.log b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/console.log new file mode 100644 index 0000000000000000000000000000000000000000..cef18d419f32ee9dd8cb3b4b6a117d1aefb67274 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/console.log @@ -0,0 +1,22 @@ +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:27:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:27:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/config.yaml +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.json +[02-20 03:28:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:41:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:41:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:41:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:41:22|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/debug.log b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b6527b49665084199abf002a7f37adfa49bf6ce4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/debug.log @@ -0,0 +1,353 @@ +[02-20 03:27:16|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 03:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 03:27:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 03:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 03:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 03:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 03:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 03:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 03:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 03:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 03:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 03:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 03:27:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 03:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 03:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 03:27:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 03:27:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 03:27:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 03:27:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 03:27:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 03:27:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 03:27:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 03:27:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 03:27:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 03:27:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 03:27:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 03:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 03:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 03:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 03:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 03:27:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 03:27:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 03:27:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 03:27:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:27:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:27:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 03:27:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 03:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:27:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 03:27:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:27:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:27:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 03:27:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.15 s +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.66 s +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.67 s +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.67 s +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 03:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 03:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.97 s +[02-20 03:27:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:28:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:28:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 03:28:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/config.yaml +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 03:28:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the person picks up the 'Mysteries Unexplained' book one last time, opens it to display the white pages, and cleans it with the pink cloth. They then place all the books down on the floor,整理 them neatly before the video ends." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.json +[02-20 03:28:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:28:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.mp4 +[02-20 03:28:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 03:28:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 03:28:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 03:28:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 03:41:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 03:41:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:41:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:41:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:41:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 03:41:22|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..efa6261cd31e8235dd627c7cd083f57e3a9eea12 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with the person整理 books on the wooden floor in a cluttered room. They pick up a blue hardcover book titled 'Britannica World & Europe,' which features a colorful globe image on the front and gold lettering for the title. The person opens the book, revealing its white pages, and wipes it with a pink cloth.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da8338710fcb93b161b457db8c10c6ef113cec3f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6aed462d1f136e363ad1ec3f6953ed7ee59a076fe2a5dee73daa081dc13555a +size 1144544 diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..7b3c899e36a07dd3465150884bfe2d278e0213a4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Next, the person places the 'Britannica World & Europe' book down and picks up another book titled 'Mysteries Unexplained.' This book has a purple cover with yellow text. They open it, showing its white pages, and clean it with the pink cloth.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d03e60e9c6c4d2e5eedc5e8594c8abd33fa3a2d6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12248afc62b0f6f0ad540502164e9ed80dd81b84ed157a03d3373056559f7ac8 +size 1180338 diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d48a4be6e47acdff5f04f6da9d0076ca492f2fae --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person then picks up a third book with a beige cover and visible text on the spine. They open it, displaying the inside pages, and wipe it with the pink cloth. The camera captures these actions from various angles, focusing on the details of the books.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da906a1137dd8398a5237408b01e3716e882c93c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7eeb4a67453a894fbaeb02980598c35e6ded2847dbc5f38d4ea43a84b4c811b7 +size 1379568 diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..61fed9da2e01915c1f75c5484012d58a3d5fa358 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Continuing, the person closes the beige-covered book and places it down. They then pick up the 'Britannica World & Europe' book again, open it once more to show the inside pages, and wipe it with the pink cloth. The camera provides close-up views of the book's pages and the cleaning process.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8103b2040775af6bd67f84297cfc605512524255 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:264609784488927ed21bdd367e8cbb5f781be472745f27cd36d8536ea90681b0 +size 1840830 diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..5b83a6c4dcf01af74014f2ff439792aec8919078 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the person picks up the 'Mysteries Unexplained' book one last time, opens it to display the white pages, and cleans it with the pink cloth. They then place all the books down on the floor,整理 them neatly before the video ends.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d2a64805153617612837f094203203a49778b6c8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:708b0cd39e7f285c51da252d2065bacaf38e6b851778588764e0e40e8488b80b +size 1511253 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58969ecfa745d2db12bc02d20b8f8fd3e8717913 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/console.log b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/console.log new file mode 100644 index 0000000000000000000000000000000000000000..65fb1f0d83c7bf364264481e6170476136305849 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/console.log @@ -0,0 +1,22 @@ +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 05:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 05:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/config.yaml +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.json +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 05:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:55:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:55:54|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/debug.log b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..01cf9d0f1797e7a8243541e654d666848a84fb6d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/debug.log @@ -0,0 +1,353 @@ +[02-20 05:41:40|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 05:41:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 05:41:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 05:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 05:41:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 05:41:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 05:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 05:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 05:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 05:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 05:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 05:41:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 05:41:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 05:41:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 05:41:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 05:41:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 05:41:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 05:41:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 05:41:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 05:41:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 05:41:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 05:41:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 05:41:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 05:41:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 05:41:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 05:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 05:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 05:41:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 05:41:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 05:41:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 05:41:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 05:41:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 05:41:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 05:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 05:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 05:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 05:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 05:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 05:41:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 05:41:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 05:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 05:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 05:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 05:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 05:41:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 05:41:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 05:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 05:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:41:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 05:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 05:41:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 05:41:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 05:41:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 05:41:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-20 05:41:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 05:41:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 05:41:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 05:41:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 05:41:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 05:41:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 05:41:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 05:42:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 05:42:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 05:42:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.90 s +[02-20 05:42:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 05:42:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 05:42:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/config.yaml +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 05:42:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The video wraps up with a series of wide shots of the baseball field, emphasizing the expansive layout and the serene environment. The user's shadow is visible on the ground, indicating their presence and activity. The final frames show the ball in mid-air, heading towards the outfield, with the camera capturing the full scope of the field and the beautiful sky above." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.json +[02-20 05:42:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 05:42:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.mp4 +[02-20 05:42:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 05:42:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 05:42:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 05:42:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 05:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 05:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:55:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:55:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:55:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 05:55:54|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0d24a3caf46b551f835fea77c677cb0899192712 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a panoramic view of a baseball field, capturing the dirt infield, green outfield, and towering light poles under a clear blue sky. The camera, positioned from the user's point of view, pans around the field, revealing the chain-link fence and empty bleachers. A red, white, and blue baseball bat is introduced, initially held by the user, setting the stage for the upcoming practice.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..609ff9800b90c0526d4e79104cecdc2e0de1c62a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70a5630803ef3888585c263e37b79d316dc38dc41fb5b75b290f01027496ae27 +size 2518169 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..60d84ec93f3b653fe64f19bad695abd5477aef9d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the camera focuses on the user placing the baseball bat on the ground near home plate, within the confines of the batting cage. The user then picks up the bat again, preparing for a swing. The camera captures the motion of the swing, with the ball being hit into the air, showcasing the dynamic action of the practice session.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f38f7f99e54e41b56a711bd62d1d6a91baa102c9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6b2331628b584658c5742018a8f09381a5cdeefb6f393bf28c660f4fb34495c +size 3095845 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..dc1b2676964fa7d3c5cb8cfbdbb0493aaa752a96 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The middle portion of the video continues with the user repeatedly swinging the bat, hitting the ball into the air each time. The camera follows the ball's trajectory, providing a sense of the distance and height achieved. The background remains consistent, with the sunny weather and surrounding trees creating a picturesque setting for the practice.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ce1443010dc1428dd9c01dd32164564a3a6b9af3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:070a06b9e415e8b16f2bad490e3cc1c25d910c49adf7b326c8cb3514da0718ef +size 1862815 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..99bb97d95de7d399ab09de56bcc1a3db80f07fbe --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the latter part of the video, the camera shifts focus back to the bat, showing it resting on the ground and then being picked up once more. The user continues to engage in batting practice, with the ball frequently seen flying towards the outfield. The camera occasionally captures the user's shadow, adding a personal touch to the footage.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3fc31f2712c189cb139d2fd38ff954e1b18f6e51 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a850d6eadcd3645ce5616c7d15b02871cbe3bfbcc0a981089485cba2803c0754 +size 2281554 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..45a9ad431f023fffa960d258b420a964eab88b17 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video wraps up with a series of wide shots of the baseball field, emphasizing the expansive layout and the serene environment. The user's shadow is visible on the ground, indicating their presence and activity. The final frames show the ball in mid-air, heading towards the outfield, with the camera capturing the full scope of the field and the beautiful sky above.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2dd7e484ece3f043195652287057668052887718 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:926dd77c453e5754899acaf1f5c66169b7e8fef94f8a3a0ad93bcfad1174f52e +size 1410668 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d007c14eab169a6d64f67be44a683e08a1bea0a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/console.log b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/console.log new file mode 100644 index 0000000000000000000000000000000000000000..227214267f5ac1d18470a51932f4e6d0563c3327 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/console.log @@ -0,0 +1,22 @@ +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:20:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:20:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/config.yaml +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.json +[02-20 01:21:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:34:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:34:35|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/debug.log b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4d1df828ab8adfc31c9ebfd4d4f9e536178ea9f5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/debug.log @@ -0,0 +1,352 @@ +[02-20 01:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 01:20:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:20:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 01:20:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 01:20:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 01:20:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 01:20:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 01:20:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 01:20:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 01:20:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 01:20:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 01:20:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 01:20:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 01:20:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 01:20:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 01:20:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 01:20:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 01:20:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 01:20:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 01:20:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 01:20:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 01:20:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 01:20:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 01:20:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 01:20:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 01:20:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 01:20:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 01:20:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 01:20:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 01:20:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 01:20:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 01:20:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:20:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 01:20:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:20:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 01:20:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 01:20:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 01:20:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 01:20:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:20:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 01:20:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 01:20:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 01:20:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 01:20:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 01:20:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 01:20:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 01:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 01:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 01:20:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 01:21:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 01:21:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 01:21:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.14 s +[02-20 01:21:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 01:21:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 01:21:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 01:21:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/config.yaml +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 01:21:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a nighttime city street scene, where a person in a red shirt and black backpack is walking along the sidewalk. The camera follows them from behind, capturing the bustling environment with bright lights, pedestrians, and parked cars. As they move forward, the perspective shifts to show their side profile, revealing more details of the lively street, including a 'Cruz Verde' pharmacy and various storefronts. The person continues walking past a red car and a green trash bin, with the camera maintaining focus on their movement amidst the dynamic urban backdrop." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.json +[02-20 01:21:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 01:21:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 01:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 01:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 01:34:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 01:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 01:34:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 01:34:35|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0325ee0c43f07d228a757e12c6f635af56534544 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a nighttime city street scene, where a person in a red shirt and black backpack is walking along the sidewalk. The camera follows them from behind, capturing the bustling environment with bright lights, pedestrians, and parked cars. As they move forward, the perspective shifts to show their side profile, revealing more details of the lively street, including a 'Cruz Verde' pharmacy and various storefronts. The person continues walking past a red car and a green trash bin, with the camera maintaining focus on their movement amidst the dynamic urban backdrop.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3c422dd26f50ed12e1fb56aea3358c5678003fd4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9032faa5ca9ba7e3dd87fa393d9a74599a2d645530ef794d1ebe1d1ad99f556e +size 1521716 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0817da3efee22e07a9f6ac367fc83f246313b53 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/console.log b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c0431abbb0de53d00ab422acb774f57c99bdf29e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/console.log @@ -0,0 +1,22 @@ +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/config.yaml +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.json +[02-20 06:05:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:20:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:20:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:20:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:20:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/debug.log b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f8e277166de046d134dd37aa3f39d0d7590dd620 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/debug.log @@ -0,0 +1,353 @@ +[02-20 06:04:49|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 06:04:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:04:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 06:04:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 06:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 06:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 06:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 06:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 06:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 06:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 06:04:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 06:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 06:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 06:04:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 06:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 06:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 06:04:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 06:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 06:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 06:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 06:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 06:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 06:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 06:04:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 06:05:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 06:05:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 06:05:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 06:05:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 06:05:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:05:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 06:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:05:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 06:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:05:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 06:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 06:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 06:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 06:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 06:05:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:05:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 06:05:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:05:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 06:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 06:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.49 s +[02-20 06:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:05:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:05:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/config.yaml +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 06:05:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The video concludes with the person still grinding the second pipe, demonstrating the repetitive and precise nature of the task. The camera captures the continuous motion of the grinding wheel and the careful handling of the pipe. The person's hands, adorned with a chain bracelet, are prominently featured as they complete the polishing process. The video ends with the person focused on the task, highlighting the dedication and skill involved in metalworking." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.json +[02-20 06:05:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:05:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.mp4 +[02-20 06:05:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 06:05:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 06:05:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 06:05:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 06:20:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 06:20:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:20:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:20:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:20:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 06:20:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ab62f2e331789222992308be4467369b3a5ea6da --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a person holding a silver metallic pipe with a 90-degree bend, which is partially covered in plastic wrap. The pipe is placed horizontally on a wooden block for stability. The person, wearing ripped blue jeans and a white long-sleeve shirt, prepares to use an angle grinder. The camera provides a close-up view from above, showing the person's hands as they position the grinder against the curved section of the pipe.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..72193dc25b266e5d04684845f56d1e4dee243e02 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f84a76521535bae89307d45c18f0a354db5a04789e14ce208ae36902da0eb5ae +size 1406838 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dcf2c400a9bd3713343361b80150eef7a2a93e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the person starts the angle grinder, the camera shifts to a side view, capturing the motion of the grinding wheel as it makes contact with the pipe. The person rotates the pipe slowly to ensure an even polish around the entire bend. The background reveals a tiled floor with a pair of black flip-flops and several other pipes lying nearby. The camera angle changes frequently, offering different perspectives of the grinding process.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fe1bbfa428a7a64e58a8a56d7031bd448ab46c59 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c65ae75aa22d75b046fff5d84d3f1394f22db8685fe2848cba837efe4eb43c6 +size 1490128 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e3da57fdb393cc64db730cb648065febe5d42418 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person continues to grind the pipe, adjusting their grip and the angle of the grinder to achieve a smooth finish. The camera captures the detailed work, focusing on the interaction between the grinding wheel and the metal surface. The person occasionally pauses to inspect the pipe, ensuring the desired level of polish is achieved. The surrounding area remains consistent, with the wooden block supporting the pipe and the other materials in the background.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17c9f2051cd9438f527f0669e25f9477bbfdd53e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bdded949f687870457e91efed1de145e7e74ed24ddd5bee9f8fcd92bcecc8ef +size 1312070 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..0263df0c45a790688a070cb66bb929d54c54ee5f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"After completing the grinding on the first pipe, the person places it aside and picks up a second similar pipe. The camera follows the movement, showing the person positioning the new pipe on the wooden block. The grinding process is repeated with the same meticulous attention to detail, as the person rotates the pipe and operates the angle grinder. The camera angle shifts to provide a clear view of the ongoing work.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..be4d0f55e57fed9ca7edea24bc0172d4aa83585b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b03b85be6edd1cce0e5df50dc0a1f8bf2c0347c818a138144727e99c054faa27 +size 1428438 diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..a1f6992e7c62e1c7cc7474ee29da7412b375457c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with the person still grinding the second pipe, demonstrating the repetitive and precise nature of the task. The camera captures the continuous motion of the grinding wheel and the careful handling of the pipe. The person's hands, adorned with a chain bracelet, are prominently featured as they complete the polishing process. The video ends with the person focused on the task, highlighting the dedication and skill involved in metalworking.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b74f4242c835ca65f35b7c529711b8ecf9468caf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e09aa93a74299b7319112757cc6798e347c7dfa0ff11a43a838dd805caa19957 +size 1400289 diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a267fca54bc7ce2805bd43afce093d73500b6a0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/console.log b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..432cf218f85b8e75d6faf9f9f262be3a2e97ffe4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/console.log @@ -0,0 +1,22 @@ +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:31:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:31:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/config.yaml +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.json +[02-20 03:31:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:46:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:46:14|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/debug.log b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0a5e8fd1c3dab13fb3536b7847ffb2239cfeb5ee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/debug.log @@ -0,0 +1,353 @@ +[02-20 03:31:04|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 03:31:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:31:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:31:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 03:31:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 03:31:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 03:31:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 03:31:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 03:31:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 03:31:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 03:31:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 03:31:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 03:31:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 03:31:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 03:31:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 03:31:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 03:31:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 03:31:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 03:31:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 03:31:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 03:31:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 03:31:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 03:31:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 03:31:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 03:31:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 03:31:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 03:31:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 03:31:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 03:31:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 03:31:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:31:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:31:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:31:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 03:31:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:31:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 03:31:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:31:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:31:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 03:31:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:31:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 03:31:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 03:31:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 03:31:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 03:31:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:31:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 03:31:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:31:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:31:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 03:31:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 03:31:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.14 s +[02-20 03:31:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:31:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:31:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 03:31:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/config.yaml +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 03:31:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The video concludes with a close-up of the bed's colorful and rumpled bedding, focusing on details like a plush toy and scattered items. The warm yellow light continues to illuminate the room, creating a cozy atmosphere. The camera lingers on the bed, emphasizing its untidy state, before the video ends." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.json +[02-20 03:31:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.mp4 +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 03:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 03:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 03:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 03:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 03:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:46:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:46:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 03:46:14|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c5cde125536a4ad8d1a3911879168514a558c128 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts in a warmly lit bathroom with a white ceramic toilet visible near the sink and mirror. The camera moves around, capturing the toilet from different angles, including a close-up of the tank and flush mechanism. A person briefly appears in the background, walking towards the door. The scene transitions to a hallway with wooden doors, where the same person is seen walking away from the bathroom.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3be503f208a6f31540649b55051e9d0fea791e26 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bce9e342d100b5fa90105ef42f3982a3dfd26492a43ff5aa59b788e8e56a45fa +size 750821 diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..d6663d8583683d0d3dc01a062b94efa9f8cb864a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"The camera follows the person into a dimly lit bedroom with a bed covered in colorful, rumpled bedding. The room has warm lighting and features a few framed pictures on the walls. The camera explores the bedroom briefly before cutting back to the bathroom, focusing again on the white toilet from a slightly different perspective.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0801db4ff2bf8e0617d166bc1b02d68d98e2527b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae2dbb6dd3ea4fb615256e18c44f2e2894562ba0909102ff4f6cf92729c7f679 +size 1101654 diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..3a6869a24b9b1670daa65deb34096bfc8fbb8a8e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The video returns to the bedroom, where the camera lingers on the bed's messy bedding, capturing details like a plush toy and scattered items. The warm yellow light illuminates the room, highlighting the disarray. The camera pans around the bed, emphasizing its untidy state.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..91ab7138e0d04824c4bd9b47ec883df2c03a0a61 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bec522497d2b3810b9987bfbee581cff584a140e0b56ddaae117df718439db2e +size 992607 diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a504b44c946ad9315a98ec10af05c8cbeec4192f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The scene continues in the bedroom, with the camera moving closer to the bed, showcasing the vibrant and chaotic arrangement of pillows and blankets. The warm lighting casts a cozy glow over the room, while the background remains softly lit. The camera captures various angles of the bed's disarray.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0242eb12523e4555acb7abdb462ed23d76de2f94 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb8d46211d51135b10bebd6b281bfef26204fca29c829c89d62e653fa5065c9 +size 905950 diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..5e27b79b4019242bf61288fb86bfc3e52b3dc721 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with a close-up of the bed's colorful and rumpled bedding, focusing on details like a plush toy and scattered items. The warm yellow light continues to illuminate the room, creating a cozy atmosphere. The camera lingers on the bed, emphasizing its untidy state, before the video ends.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85b0ddda19717b01b35478224ea76e0b8443f963 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff507b1f2ce8b241993b2fb98641bc4826cc6854ac505624f7dded6c27f09b6a +size 880058 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22c4e61171e97e1b7c3bce88f467bd1d859d314a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/console.log b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/console.log new file mode 100644 index 0000000000000000000000000000000000000000..69f5444fb9050b4420233dc64c639726fad08b2c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/console.log @@ -0,0 +1,22 @@ +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/config.yaml +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.json +[02-19 23:12:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:25:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:25:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:25:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:25:26|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/debug.log b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2d2beec530c0c13dd85573b2711e656747e588b2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/debug.log @@ -0,0 +1,353 @@ +[02-19 23:11:15|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 23:11:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:11:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:11:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 23:11:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 23:11:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 23:11:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 23:11:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 23:11:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 23:11:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 23:11:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 23:11:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 23:11:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 23:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 23:11:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 23:11:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 23:11:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 23:11:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 23:11:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 23:11:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 23:11:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 23:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 23:11:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 23:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 23:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 23:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 23:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 23:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 23:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:11:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 23:11:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 23:11:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.10 s +[02-19 23:11:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:12:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:12:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/config.yaml +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 23:12:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the person gives the sink area a thorough cleaning, wiping down the stainless steel surface and ensuring all corners are spotless. They rinse the sponge one last time and place it aside. The video ends with the kitchen looking organized and clean, with the white plate with the blue floral pattern neatly stored in the cabinet.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.json +[02-19 23:12:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.mp4 +[02-19 23:12:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 23:12:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 23:12:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 23:12:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:25:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:25:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:25:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:25:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:25:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:25:26|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e79923c1ad676bdd37410f073d5716ce8f947536 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of a cluttered kitchen sink area, featuring a white plate with a blue floral pattern that has food remnants on it. The person's hands are seen moving around, picking up items from the sink and starting to organize the space. The background shows a window with potted plants and various kitchen appliances, including a pressure cooker.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..099921baab5a5d89d12ed7b8427a98026050f07a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34e55ec8f573bc72544be75fd3183b00a9789ed0da9399ea0a1c0577b53ec0ad +size 809513 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..a01f3f74dec658dc374bbaa714509c57254dc47d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"The person retrieves utensils from a drawer near the stove and places them on the counter. They then move to the sink area, washing dishes including the white plate with the blue floral pattern. The person carefully places the cleaned plate into a cabinet above the counter, alongside other plates and mugs. The surrounding area includes a wooden cutting board and various kitchen containers.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1c285f24cbab03e71cb3785c1a8cd27f63d4779a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bf2f52f2b74fc16b1dc4ce8f331d57c3c229ca3eab197a66a85ea4a66dc8e87 +size 1868367 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a2a7f297dcde44fa72e05d416068b4dff036a8db --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Continuing from the previous scene, the person focuses on washing more dishes at the sink, including a small cup and a spoon. They place these cleaned items into the cabinet, ensuring everything is neatly stored. The kitchen remains well-lit with natural light, and the window with potted plants is still visible in the background.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49f4d5ba4b77f90853ec98ff3e734c87ecf47064 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:021fe2de16e147b55a7a9678f9e6fe3623aa2b0b55ee327ea0471161c7ed9ad3 +size 1361270 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..5b0707c3f4d07d8e0b39659e0afddf17d9129420 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The person proceeds to wipe down the sink area with a sponge, ensuring all surfaces are clean. They then rinse the sponge under running water and continue to tidy up the sink area. The white plate with the blue floral pattern is seen being placed back into the cabinet, completing the organization process.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ed227129e6147749fc1f1d8b045ecfb49f33ce0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da7761e43403de3131be1fa357c485ce822ebff78aad0c9cb283ea69ae4b9bd +size 1591324 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..2b8e8340c0a17ad7cb72129440a571725737e501 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the person gives the sink area a thorough cleaning, wiping down the stainless steel surface and ensuring all corners are spotless. They rinse the sponge one last time and place it aside. The video ends with the kitchen looking organized and clean, with the white plate with the blue floral pattern neatly stored in the cabinet.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a338b1a565f2a3646cae75189bd99ac70460fe1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec54f24670a05407679b18538bdcb7679ae581c78ed1ccf0a04964c594903c1c +size 1054723 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e314ca0ca4edb95a15d93888bc1ef9605bcad4ad --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/console.log b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/console.log new file mode 100644 index 0000000000000000000000000000000000000000..e94c061082c6202e682821eb8f120b13f3afc618 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/console.log @@ -0,0 +1,22 @@ +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:22:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:22:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:22:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:22:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/config.yaml +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.json +[02-20 17:23:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:36:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:36:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:36:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:36:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/debug.log b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dd096a827acc8850635f085a5a39fd94f1c9fc9e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/debug.log @@ -0,0 +1,353 @@ +[02-20 17:22:27|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 17:22:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:22:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 17:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 17:22:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 17:22:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 17:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 17:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 17:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 17:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 17:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 17:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 17:22:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 17:22:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 17:22:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 17:22:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 17:22:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 17:22:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 17:22:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 17:22:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 17:22:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 17:22:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 17:22:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 17:22:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 17:22:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 17:22:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 17:22:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 17:22:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 17:22:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 17:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 17:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 17:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 17:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 17:22:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 17:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 17:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 17:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 17:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 17:22:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 17:22:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 17:22:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:22:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 17:22:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:22:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 17:22:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 17:22:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 17:22:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 17:22:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 17:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 17:22:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 17:22:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 17:22:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 17:23:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 17:23:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 17:23:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.19 s +[02-20 17:23:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 17:23:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 17:23:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/config.yaml +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 17:23:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='The video concludes with the person sweeping the remaining areas of the patio, ensuring the entire surface is clean. The red broom is seen moving back and forth, gathering all the debris into a pile.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.json +[02-20 17:23:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 17:23:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.mp4 +[02-20 17:23:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 17:23:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 17:23:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 17:23:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 17:36:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 17:36:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 17:36:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 17:36:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 17:36:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 17:36:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcddf0bcaea619fd7166874785f3de8f7baf234 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a first-person perspective of someone walking down a narrow outdoor corridor with beige walls, casting shadows from nearby plants. The person passes by a small gate and enters a garden area with brick pathways and lush greenery.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..749127e0ef6fab9729b2cc88c4b642e654de702c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c8f049aa8fcac9cf2fc98272c812b512bf2b7c09d0af9fd2521881ed10637f7 +size 2277735 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..cc9dacc4e7ac67b67b6e2688fce5b17ae2060716 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"After exiting the garden, the person returns to the patio, where they pick up a red broom with a long wooden handle and black grip from a storage area near the house wall.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a951d48966dd6c24ac11d46a262496766ac2d279 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47b47a2ce7412323c7b1a355739a8375803cd10f0f8cbab8f8146c695b0e401e +size 2976371 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..ca6d036d7f552ea0698557a58821a662d1265ee3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person begins sweeping the tiled patio floor with the red broom, moving around a glass table with pink plates and several chairs. The sweeping motion is methodical, covering the entire patio surface.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..660f44703b5ebc4027315823f21044b7f40667fa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eea65e5f9b2759da17951a12197b415fdf9d450ded016acbb8885ed090092f55 +size 2238051 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..459b2300618689c195ac28e9d51eda3fc5ab3162 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the person continues sweeping, they focus on different sections of the patio, including near the sliding glass door, around the edges, and along the border with the grassy area. The red broom effectively collects dirt and debris.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9669172e50084bed1cb0fee5268e1868bc10a08e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a366c8ab8d887eb72194b42115c348b7a6a0d3f8a5e0db5e75d878cc7ca72c46 +size 2573818 diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..6bdeb81d135ad26112e0998001f98c31562270a6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with the person sweeping the remaining areas of the patio, ensuring the entire surface is clean. The red broom is seen moving back and forth, gathering all the debris into a pile.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..83b7d210e28821b1f22b90a56648255f8618ae02 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df132e283f3f1c1e394d09c89bc9f61a6b5fcb613ba45d7486dc4d48941cb266 +size 2618040 diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9aecad8b35b6b670180a44348582ce0b2cef4724 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/console.log b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/console.log new file mode 100644 index 0000000000000000000000000000000000000000..9cd962c9e9b50010fc59cb466ea25439f8c349b5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/console.log @@ -0,0 +1,22 @@ +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/config.yaml +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.json +[02-20 10:11:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:24:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:24:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:24:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:24:34|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/debug.log b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..84e62db6b9401e8ead755d3841171bdacf794792 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/debug.log @@ -0,0 +1,353 @@ +[02-20 10:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 10:10:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:10:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 10:10:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 10:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 10:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 10:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 10:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 10:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 10:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 10:10:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 10:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 10:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 10:10:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 10:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 10:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 10:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 10:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 10:10:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 10:10:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 10:10:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 10:10:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 10:10:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 10:10:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 10:10:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 10:10:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 10:10:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 10:10:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 10:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 10:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 10:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 10:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 10:10:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 10:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 10:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 10:10:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:10:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 10:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:10:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 10:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:10:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 10:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:10:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 10:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 10:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.59 s +[02-20 10:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:11:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:11:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 10:11:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/config.yaml +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 10:11:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the person continues to smooth and adjust the tarp, focusing on the edges and corners of the frame. A child is visible in the background, walking around the construction site, while the person remains concentrated on their task.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.json +[02-20 10:11:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:11:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.mp4 +[02-20 10:11:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 10:11:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 10:11:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 10:11:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 10:24:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 10:24:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:24:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:24:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:24:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 10:24:34|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..9c424c1dc7ef03be1a3fe05a0ea77bba7a780d5c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up view of a person securing a black tarp over a rectangular wooden frame filled with rocks. They use a tool to fasten the tarp along the edges of the frame, ensuring it is tightly wrapped around the structure.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70f993cd46b3f1fa37f6d52f525a672585046063 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3da6aa564f9aef1728693efdd860f4d34f4c74e341c6dacbcb55a779a7725e8 +size 972506 diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9dc9c9a807073ec74827c69a0083337ffb9040ce --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"After securing the tarp, the person removes it partially to reveal the rocks inside the frame. They then step away from the frame, walking across the construction site, which is covered in gravel and surrounded by wooden structures.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d9ba53f41e04a1853cbf9c70513fae1fd434482 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a266e8f47ce3520c8f47a689e435a046b095e0fa9e6b5fb92c3e708ea528f4e +size 1589913 diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..2de5505b722a672f08c88db2b9642e1401de01b3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The person approaches a wooden box near a parked car, opens it, and retrieves an object. They examine the object briefly before walking back toward the wooden frame, carrying the item with them.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb98201c571dd178605f4fe3faf4410f4b9052a7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6077b0caf76dfee1d674fa59e284604f2ca55119073bc502df3605ab3265610b +size 1489382 diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9f531862fafe67269eec3da54980629766755f7b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Upon returning to the frame, the person kneels down and begins readjusting the black tarp, smoothing it out over the rocks. They make several careful movements to ensure the tarp is properly positioned and secured.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d0383ae01767ea80d973bef8517b170cce6b7000 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e042fa18962d836f8212a04fbc8d47f1eb8612c8f55fb00e14c6959f96c1123 +size 1319466 diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..2803ffc28b4155c846422fe27065bfd948caafe9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the person continues to smooth and adjust the tarp, focusing on the edges and corners of the frame. A child is visible in the background, walking around the construction site, while the person remains concentrated on their task.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0123476f25f0367d3f75717fc672126c9a7a8f9f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aef94cebd9a0f637ad9dec82c01328faf1246251f6bea1a8b989960c47817d3 +size 1047591 diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/config.yaml b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..449ac8eced401f03172541c29a14632f1c31f7d3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/console.log b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..a86c108f1d7696701198f356bdb77739fd01b262 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/console.log @@ -0,0 +1,22 @@ +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/config.yaml +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.json +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:37:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:37:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:37:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:37:21|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/debug.log b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3877b73e76c98acc143b2fb2a48fc7c17bd6af95 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/debug.log @@ -0,0 +1,353 @@ +[02-19 21:22:09|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 21:22:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:22:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 21:22:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 21:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 21:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 21:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 21:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 21:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 21:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 21:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 21:22:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 21:22:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 21:22:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 21:22:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 21:22:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 21:22:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 21:22:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 21:22:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 21:22:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 21:22:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 21:22:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 21:22:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 21:22:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 21:22:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 21:22:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:22:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:22:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 21:22:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:22:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 21:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:22:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 21:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 21:22:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:22:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:22:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:22:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 21:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 21:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.39 s +[02-19 21:22:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:22:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 21:22:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/config.yaml +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 21:22:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the concluding segment, the person completes the assembly by attaching the last few screws and making final checks to ensure everything is secure. The camera provides a comprehensive view of the fully assembled cabinet, showcasing the clean lines and precise fit of the white panel. The video ends with a shot of the completed furniture piece, highlighting the successful assembly process.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.json +[02-19 21:22:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:22:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.mp4 +[02-19 21:22:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 21:22:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 21:22:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 21:22:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 21:37:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 21:37:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:37:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:37:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:37:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 21:37:21|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_0.json b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..dcb9fe76915f7e2e2f37e94d15e9284766a7af5d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up shot of a person's hand, adorned with blue nail polish and a pink watch, placing a white rectangular furniture panel into a partially assembled cabinet frame. The panel features a grooved design on one side and a smooth surface. The person carefully aligns the panel, ensuring it fits correctly within the frame.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dc39bf7b08ed2137113aa761803b7cb6253d3a6f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052647382760fc15923a95d0a053a57cd512cd14b69fbb58c24a417529b95275 +size 770821 diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_1.json b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..78ac612ac71bb83b775f05f2b11487a985de7ecb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Next, the person picks up a smaller white component from the carpeted floor and attaches it to the inside of the cabinet. They then proceed to pick up screws from a plastic bag and begin inserting them into pre-drilled holes on the panel. The camera captures detailed close-ups of the screwing process, showing the person's hands working methodically to secure the panel in place.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..148be0da7cb068e94be5cf3796294237afc6bd45 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58c53dbb9f2c279ac8e46d2287b8c7607d60ebcc38fc797eba47730fc378b576 +size 2692121 diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_2.json b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..ecc9fd1379328b990039928978eb52e49731e7df --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"As the assembly progresses, the person continues to tighten screws into the white panel, focusing on ensuring each one is properly aligned and securely fastened. The camera angle shifts to provide top-down views of the cabinet, highlighting the precision required for this step. The person occasionally glances at the instruction manual on the floor, following the diagrams and steps closely.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2108919f5f3f1dda442f084326c3d57506a6bb73 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68e87f9eb90e4b688d9e2e556e6920a62af9c1f6efb05a93639b0162474da098 +size 1342788 diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.json b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..8cb50e2a35c76e11d141128869699655866a0885 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The video then shows the person making final adjustments to the white panel, checking its stability and alignment from various angles. The camera captures the person's hands as they use a screwdriver to make any necessary tweaks, ensuring the panel is perfectly fitted within the cabinet frame. The instruction manual remains visible, serving as a constant reference throughout the process.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f2c003f8fd272f62dfbb5b296b1c7fac4cb18b98 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:937c6055377c2705ae7d250125bf905969f24503c0e4b3bd2c6b02ade1fa6e30 +size 1369417 diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.json b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..97b2b1f9762e3be367ffaac8a31c26aeb4613c97 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the concluding segment, the person completes the assembly by attaching the last few screws and making final checks to ensure everything is secure. The camera provides a comprehensive view of the fully assembled cabinet, showcasing the clean lines and precise fit of the white panel. The video ends with a shot of the completed furniture piece, highlighting the successful assembly process.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f5d9e4addd3ce7d8083006d73e94c4c265941981 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3be85188c723f135888c6cf499c716bf358070a2f65164aec4e94157616ac4de +size 1219745 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ef573b89cc33fce02a3647c85496359f039aa88 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/console.log new file mode 100644 index 0000000000000000000000000000000000000000..2b6cb3d7b3bd517d50d9e900db2ec97bbbad33ef --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/console.log @@ -0,0 +1,22 @@ +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:59:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:59:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/config.yaml +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.json +[02-19 22:00:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:13:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:13:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:13:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:13:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c11787c76aaccff849e13a191959f172e5498181 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/debug.log @@ -0,0 +1,353 @@ +[02-19 21:59:40|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 21:59:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:59:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:59:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 21:59:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 21:59:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 21:59:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 21:59:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 21:59:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 21:59:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 21:59:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 21:59:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 21:59:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 21:59:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 21:59:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 21:59:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 21:59:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 21:59:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 21:59:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 21:59:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 21:59:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 21:59:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 21:59:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 21:59:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 21:59:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 21:59:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 21:59:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:59:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:59:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 21:59:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:59:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:59:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:59:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 21:59:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:59:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 21:59:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:59:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:59:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 21:59:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:59:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:59:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:00:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 22:00:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 22:00:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.06 s +[02-19 22:00:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:00:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/config.yaml +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 22:00:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final moments of the video, the man and woman are still standing in the alleyway, engaged in their conversation. The man, with his white and pink jacket, black pants, glasses, and backpack, appears to be concluding his point, while the woman, in her black leather jacket, blue top, and black pants, listens intently. The rough stone walls and overgrown vegetation remain unchanged, and the dappled sunlight continues to cast shadows on the ground, maintaining the atmospheric setting of the scene.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.json +[02-19 22:00:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:00:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.mp4 +[02-19 22:00:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 22:00:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 22:00:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 22:00:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 22:13:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 22:13:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:13:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:13:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:13:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 22:13:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..71deb424cc8a6f908f513d5205783724061c34c9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a man and a woman walking down a narrow, overgrown alleyway. The man, wearing a white and pink jacket, black pants, glasses, and a black backpack, leads the way, while the woman, dressed in a black leather jacket over a blue top, black pants, and with her hair in a ponytail, follows closely behind. The alley is flanked by rough stone walls with vines and plants growing along them, and sunlight filters through the foliage above, casting dappled shadows on the ground.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24da03ed2ae14d70bf2975787b7daa3fbda76487 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cada6099b716130bb4e9d70e1c65ad19c98b1bb8871137327d887f3ce685fae +size 1551220 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..4759a3ce1f29eb9d79feadb3177ff0614bf37b29 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As they continue walking, the man turns around to face the woman, who has stopped beside him. Both individuals are now standing in the alleyway, facing each other. The man, still wearing his white and pink jacket, black pants, glasses, and backpack, begins to speak, gesturing slightly with his hands. The woman, in her black leather jacket, blue top, and black pants, listens attentively, her posture relaxed but focused.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e46e13e4b79e7a23b743b6e27dc729a8909b4fa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dfe372876b1d713fffa53901d05cc59f4badd0e0650bca312138018ff1aeb62 +size 1467202 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..037dcee0ece849d1af9ca3a7d920f2ff6327fb35 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The conversation between the man and woman continues as they stand in the alleyway. The man, with his white and pink jacket, black pants, glasses, and backpack, maintains eye contact with the woman, his expression serious as he speaks. The woman, in her black leather jacket, blue top, and black pants, nods occasionally, indicating her engagement in the conversation. The rough stone walls and overgrown vegetation remain prominent in the background, with dappled sunlight continuing to illuminate the scene.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3cf2388f81ea57692efe728e390b15d0b5e5ad8f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4effd55d09072169c30e96640628b8bb6f07affddd91302e128841a8be526579 +size 908323 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..f360febe2b2e11340c68547249542e46b3dd2e56 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The man and woman remain in the same position, deeply engrossed in their conversation. The man, still wearing his white and pink jacket, black pants, glasses, and backpack, continues to speak, his tone suggesting a serious discussion. The woman, in her black leather jacket, blue top, and black pants, responds, her body language indicating attentiveness and interest. The alleyway's rough stone walls and overgrown vegetation provide a consistent backdrop, with sunlight filtering through the foliage above.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37ee860ecb2feab45c21f40ddeb39c14823bf28d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33fe616b729af0054f490e733e375ccb0c967e38afe379aeaaf3230965ccccc2 +size 655637 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ce77a977a1f02531f35acf44700816109eb440ee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments of the video, the man and woman are still standing in the alleyway, engaged in their conversation. The man, with his white and pink jacket, black pants, glasses, and backpack, appears to be concluding his point, while the woman, in her black leather jacket, blue top, and black pants, listens intently. The rough stone walls and overgrown vegetation remain unchanged, and the dappled sunlight continues to cast shadows on the ground, maintaining the atmospheric setting of the scene.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a66dd6dfc2c0bae8b5ce54294d17d7035b9e71a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b26da6799b5b626a7d0c0115e39302ff7b05f662e0bc0e5312df3cf65743363 +size 566468 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9672eefab4bf419ceb34d2436a7755faa6667cc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/console.log new file mode 100644 index 0000000000000000000000000000000000000000..2aa00ed1ea8487a70aebef1fb3d24d267228294a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/console.log @@ -0,0 +1,22 @@ +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/config.yaml +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.json +[02-20 21:13:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:28:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:28:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:28:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:28:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b45095d3a3a6610273bd71f3f15516ca97c3edef --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/debug.log @@ -0,0 +1,353 @@ +[02-20 21:12:51|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 21:12:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:12:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 21:13:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 21:13:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 21:13:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 21:13:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 21:13:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 21:13:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 21:13:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 21:13:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 21:13:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 21:13:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 21:13:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 21:13:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 21:13:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 21:13:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 21:13:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 21:13:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 21:13:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 21:13:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 21:13:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 21:13:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 21:13:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 21:13:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 21:13:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 21:13:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 21:13:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 21:13:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 21:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 21:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:13:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 21:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 21:13:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 21:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 21:13:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 21:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 21:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 21:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 21:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.69 s +[02-20 21:13:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 21:13:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 21:13:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 21:13:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/config.yaml +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 21:13:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final moments of the video, the man remains in the same position, his profile facing right. The shifting light continues to cast dynamic shadows across his face, emphasizing his features. His dark, curly hair and light stubble remain unchanged, and the striped collar of his shirt is still visible. The textured wall in the background provides a consistent, dimly lit environment. The camera remains steady, ensuring the man's identity and the overall composition of the scene remain intact." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.json +[02-20 21:13:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.mp4 +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 21:13:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 21:13:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 21:13:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 21:28:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 21:28:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 21:28:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 21:28:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 21:28:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 21:28:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3afa0497206903d92551f070cffc4f295af74253 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a close-up profile shot of a man with dark, curly hair and light stubble. He is wearing a light-colored, textured jacket over a collared shirt with thin vertical stripes, the collar visible at the neckline. The setting is a dimly lit room with a rough, textured wall in the background. Soft, shifting light begins to cast dynamic shadows across his face, creating a subtle interplay of light and shadow that highlights his features. The camera remains stationary, focusing on the man's profile, ensuring his appearance and the spatial consistency of the background are maintained.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39b3e6ff4b9adf59bb92a2cc16b0c5f67185cac3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18fd069d23a9d15caae24da1648f3e90cbae959cb3b6baf0e4ac887cd7248997 +size 1053484 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..204acb1ef589db9a8a2fb95c28d406526fc62f94 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the man remains in the same position, his profile facing right. The shifting light continues to create new patterns of shadow and light across his face, emphasizing his nose, lips, and the texture of his stubble. His dark, curly hair and the striped collar of his shirt remain clearly visible. The textured wall in the background stays consistent, providing a stable backdrop. The camera remains fixed, capturing the subtle changes in lighting without any movement.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6cc84010850f659867f28e95baca96bbd8a3cc13 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7c61836247be8703e75e88835a570497b2a531c08469494b11b11267437ebcc +size 806244 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d802b29a2f49e14355633dff0ba20812826cea8d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The lighting continues to shift, casting new shadows across the man's face, highlighting different aspects of his facial structure. His dark, curly hair and light stubble remain unchanged, and the striped collar of his shirt is still visible. The textured wall in the background provides a consistent, dimly lit environment. The camera remains steady, ensuring the man's identity and the overall composition of the scene are preserved.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2062ccc16506e446d250881ea3c34269bfad22cf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a607f388beff325724e74440f9361f6f80ab3c4ee83686a65fbb918558f92f17 +size 678815 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..2d929d1585391db9e055a8d1f92c069af2453722 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The shifting light continues to play across the man's face, creating a dynamic interplay of shadows and highlights. His dark, curly hair and light stubble remain consistent, and the striped collar of his shirt is still visible. The textured wall in the background remains unchanged, providing a stable backdrop. The camera remains stationary, maintaining focus on the man's profile.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b1e5f9dcb071755a373961925aaefb87a7fa87ba --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:430bfac07a2a3357387fd993ad30e365199781c0a98f6aaa33017b0b59e9182c +size 679937 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..b171d11f2906cd5767fb701398d87647492e7d2b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments of the video, the man remains in the same position, his profile facing right. The shifting light continues to cast dynamic shadows across his face, emphasizing his features. His dark, curly hair and light stubble remain unchanged, and the striped collar of his shirt is still visible. The textured wall in the background provides a consistent, dimly lit environment. The camera remains steady, ensuring the man's identity and the overall composition of the scene remain intact.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6fc7e2e170d29e0ff5b2ee983d39627e6841bcb1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b146d5b660b1413bdabd131c24955ad62e66f8284f7c3b87298dd9dae539ae77 +size 659457 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..883d0c5709ce66675dab813e9e9e0e2ba5cf550e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/console.log new file mode 100644 index 0000000000000000000000000000000000000000..585ae62d39aa083b8a31b9c590a023e5fa6ba6b8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/console.log @@ -0,0 +1,22 @@ +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 15:07:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 15:07:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/config.yaml +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.json +[02-20 15:07:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:21:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:21:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:21:15|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1330a101893a14ca1e9a32be8a54333c4a7abe31 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/debug.log @@ -0,0 +1,352 @@ +[02-20 15:07:09|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 15:07:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 15:07:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 15:07:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 15:07:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 15:07:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 15:07:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 15:07:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 15:07:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 15:07:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 15:07:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 15:07:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 15:07:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 15:07:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 15:07:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 15:07:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 15:07:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 15:07:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 15:07:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 15:07:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 15:07:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 15:07:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 15:07:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 15:07:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 15:07:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 15:07:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 15:07:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 15:07:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 15:07:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 15:07:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 15:07:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 15:07:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 15:07:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 15:07:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 15:07:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 15:07:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 15:07:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 15:07:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 15:07:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 15:07:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 15:07:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:07:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 15:07:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:07:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 15:07:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 15:07:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 15:07:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 15:07:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 15:07:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 15:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 15:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 15:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.68 s +[02-20 15:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 15:07:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 15:07:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 15:07:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/config.yaml +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 15:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The scene transitions as the central man finishes his bite and turns slightly to his left, continuing his conversation with the man on his left. The man on the right laughs and gestures more animatedly, adding to the lively interaction. The background remains filled with market activity, with new characters entering the frame, including a woman in a floral dress carrying a basket. The camera pans slightly to follow the movement of the central man as he interacts with the others, maintaining the focus on the trio amidst the bustling market.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.json +[02-20 15:07:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 15:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 15:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 15:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 15:21:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 15:21:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:21:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:21:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:21:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 15:21:15|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ac32f11b67b4dd58912b97bbb5cc69e19fa280c0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The scene transitions as the central man finishes his bite and turns slightly to his left, continuing his conversation with the man on his left. The man on the right laughs and gestures more animatedly, adding to the lively interaction. The background remains filled with market activity, with new characters entering the frame, including a woman in a floral dress carrying a basket. The camera pans slightly to follow the movement of the central man as he interacts with the others, maintaining the focus on the trio amidst the bustling market.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ed1f8a962a0ce4673a3fe6a132f36d5a8d3a6ab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:192cbd7482be811fec34444eed130993b239c0179e12ddd8503d8cb864d9c2fb +size 1412789 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ec2e7a04a75ac2539245f617dcb1fc121fc47d6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..121758863ce78d4f18ce0c0cd9544c169c572a30 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/console.log @@ -0,0 +1,22 @@ +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:18:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:18:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/config.yaml +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.json +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:32:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:32:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:32:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:32:47|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..043c84f3585e49eb064525dbdf03c381ae12a1b9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/debug.log @@ -0,0 +1,353 @@ +[02-20 02:18:35|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 02:18:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:18:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 02:18:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 02:18:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 02:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 02:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 02:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 02:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 02:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 02:18:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 02:18:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 02:18:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 02:18:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 02:18:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 02:18:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 02:18:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 02:18:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 02:18:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 02:18:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 02:18:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 02:18:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 02:18:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 02:18:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 02:18:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 02:18:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:18:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:18:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 02:18:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:18:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:18:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:18:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 02:18:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:18:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 02:18:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:18:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 02:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 02:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 02:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 02:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 02:18:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:18:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:18:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 02:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 02:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.29 s +[02-20 02:19:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:19:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:19:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/config.yaml +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 02:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the two men continue their walk along the path, maintaining their pace and position. The man on the left, in his white collared shirt and dark overcoat, and the man on the right, in his black leather jacket, remain side-by-side. The bamboo stalks and the building with windows provide a consistent backdrop. The camera maintains its following shot, capturing their final moments of the sequence.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.json +[02-20 02:19:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.mp4 +[02-20 02:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 02:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 02:19:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 02:19:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 02:32:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 02:32:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:32:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:32:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:32:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 02:32:47|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..90f82674aa5d3f39f5367f81183d5923109a5a5d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with the two men walking side-by-side at night. The man on the left, wearing a white collared shirt under a dark V-neck sweater vest and a dark overcoat, and the man on the right, dressed in a black leather jacket over a dark shirt, are seen walking along a path lined with tall bamboo stalks on the left and a building with windows on the right. The lighting is dim, creating a subdued atmosphere. The camera follows them from behind and slightly to the side, capturing their initial movement.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64afb0fbe5e21b0a1d6682069187b44771251e5a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58c98f5540e83f993948162d4987ea1150e7c394d4af0eb462dcde9d741ebc1a +size 1190660 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..61e2748f1703b527f2cedcad77d70a667e6875f7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As they continue walking, the man on the left briefly touches his face with his right hand, adjusting his collar. The man on the right remains in profile view, maintaining his pace. The background of bamboo stalks and the building's windows remains consistent. The camera continues to follow them smoothly, keeping them centered in the frame.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..feaffbe755870df9a153811a1d2bb025074135ed --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ebf3a82990cddc68fdde409d338ccb1415f6ce003f6248cde58b654b08cadfb +size 1276900 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e93aa9df050def8cf003ccb009e0e49509c6b605 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The two men proceed along the path, their pace steady and their positions unchanged. The man on the left, in his white collared shirt and dark overcoat, and the man on the right, in his black leather jacket, continue to walk side-by-side. The bamboo stalks and the building with windows provide a consistent backdrop. The camera maintains its following shot, ensuring a continuous and stable view.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05d951d1a743eeed8a37f3347f22aaf8ce6c113d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e062837bdc7643a2e1eac05b303f2c1467677af03da1ea4edfa87c1123477f6 +size 1260901 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..fd47fff5ab5014fa65cee7dde3f8d68f79ce64ca --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As they move further along the path, the man on the left adjusts his collar again, while the man on the right remains focused ahead. The background elements, including the bamboo stalks and the building's windows, remain unchanged. The camera continues to follow them from behind and slightly to the side, capturing their steady movement.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ed30018de943f98f22159dfaf455503b3369b5fd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64fb8a184bbf2c83173fdbcea8a92cde6139cb2536d5510c9bac8d5d031a1672 +size 1352205 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..973d47b4a4f41b062ea49086231d2964add6e959 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the two men continue their walk along the path, maintaining their pace and position. The man on the left, in his white collared shirt and dark overcoat, and the man on the right, in his black leather jacket, remain side-by-side. The bamboo stalks and the building with windows provide a consistent backdrop. The camera maintains its following shot, capturing their final moments of the sequence.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7fbed8aa654e006bb25f22f1336d457554845bf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884d9c923602718817dac2b218ef0fee49e47f8e49eebfca8e1c5a3c2dd5b961 +size 1318652 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..406c0ec409c2bbb3499c98bd11f87ee328661dc9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/console.log new file mode 100644 index 0000000000000000000000000000000000000000..490e4ee7adc7b104c3aaf65a097c92c6d0ee73ca --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/console.log @@ -0,0 +1,22 @@ +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/config.yaml +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.json +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 05:03:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:03:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:03:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:03:06|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f113366711d842a1c64ff8b746f6b16f3d7bf728 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/debug.log @@ -0,0 +1,353 @@ +[02-20 04:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 04:47:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:47:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 04:48:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 04:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 04:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 04:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 04:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 04:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 04:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 04:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 04:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 04:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 04:48:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 04:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 04:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 04:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 04:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 04:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 04:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 04:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 04:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 04:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 04:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 04:48:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 04:48:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 04:48:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 04:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 04:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 04:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 04:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 04:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 04:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 04:48:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 04:48:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 04:48:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:48:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:48:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 04:48:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:48:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 04:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:48:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 04:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 04:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 04:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 04:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 04:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 04:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:48:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:48:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 04:48:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 04:48:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.58 s +[02-20 04:48:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:48:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:48:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/config.yaml +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 04:48:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final moments, the camera remains steady, capturing the full-body posture of both women as they engage in their conversation. The woman in the white turtleneck sweater maintains her crossed-arm stance, her expression unwavering, while the second woman stands beside her, her body language suggesting she is listening carefully. The background elements, including the framed abstract painting and patterned wallpaper, remain consistent, providing a cohesive visual narrative throughout the video.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.json +[02-20 04:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.mp4 +[02-20 04:48:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 04:48:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 04:48:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 04:48:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 05:03:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 05:03:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 05:03:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 05:03:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 05:03:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 05:03:06|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2f13726b81c160d308907f1e14e3392e04f041ed --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up shot of a woman with long, dark brown/black wavy hair, wearing a white ribbed turtleneck sweater, silver drop earrings, and a nose ring on her left nostril. She is facing away from the camera, revealing her profile as she turns to engage with another woman who is partially visible, wearing a black jacket with a red collar. The second woman has her hair styled in a neat bun. The setting appears to be a modern indoor space, possibly an office or studio, with patterned wallpaper, a framed abstract painting, and circular ceiling lights in the background.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc1d895f35de27cab7415106fc6cc2e278f5561f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d490916dda0909c233124e6a0d4e3d99a7325321ef1aec07a2b14fa8c498bde1 +size 820431 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..05183855673acf20882f4753881487f190504b97 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the camera focuses on the woman in the white turtleneck sweater as she begins to speak. Her facial expressions shift subtly, indicating she is engaged in a conversation. The second woman listens intently, her head slightly tilted. The camera angle shifts slightly, offering a more frontal view of the first woman as she maintains her position in the frame, with the background elements remaining consistent.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c236ba2ed18fc9156f6f5418672cb4755f73f09 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5a975a7e7b773ee66ab63aa1a6bf476fcff20c9edfe5154379067459345f300 +size 961203 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0a637c81d82d9ad01ba644433c76183eb094688a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera continues to follow the woman in the white turtleneck sweater as she speaks, capturing her upper body and face in detail. Her long, wavy hair frames her face, and her neutral makeup with pink lipstick is clearly visible. The second woman remains partially visible, her attention focused on the speaker. The background includes the framed abstract painting and patterned wallpaper, providing a stable visual context for the interaction.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..011dcf0083529b50a945854d5e3d116bf383d83e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25ac757d4cb510edd1fbb50d99496616795f5c2d63ca93ab6b8657f1f8e46136 +size 899744 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..f6f65ecd33b3521d41f948a6b165f4c5eee2b6a5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The camera pulls back to reveal more of the room, showing both women standing side by side. The woman in the white turtleneck sweater has her arms crossed, her expression serious as she continues to speak. The second woman, now fully visible, stands beside her, listening attentively. The background includes a large framed abstract painting on the wall, patterned wallpaper, and a circular ceiling light fixture.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..27f9ace714a602e82412d309f7e2bc8385b6b628 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e3370d4dfca6784373c2387f9879692648b9f4a6ed6eed9cbb278c9849cc052 +size 1837286 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e1079daf45a66d00bc4a80c009faf6c9a04d0b7d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments, the camera remains steady, capturing the full-body posture of both women as they engage in their conversation. The woman in the white turtleneck sweater maintains her crossed-arm stance, her expression unwavering, while the second woman stands beside her, her body language suggesting she is listening carefully. The background elements, including the framed abstract painting and patterned wallpaper, remain consistent, providing a cohesive visual narrative throughout the video.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c123a70913c80c1d72278cea45b6c24b32af63c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6232d1aaa087bb81fe99a15f7a4b6d2844fcf78f60c2bb045f57252c02b4378 +size 1619609 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59214439bda9ef0cace848daf1209882f7732a56 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/console.log new file mode 100644 index 0000000000000000000000000000000000000000..d673780c0aba1473957952f45942dacf8a009879 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/console.log @@ -0,0 +1,22 @@ +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:07:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:07:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/config.yaml +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.json +[02-20 00:07:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:20:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:20:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:20:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:21:01|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..481da9604b13d62bf8004278272b074d41b81ac5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/debug.log @@ -0,0 +1,353 @@ +[02-20 00:06:55|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 00:07:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:07:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:07:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 00:07:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 00:07:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 00:07:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 00:07:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 00:07:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 00:07:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 00:07:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 00:07:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 00:07:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 00:07:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 00:07:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 00:07:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 00:07:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 00:07:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 00:07:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 00:07:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 00:07:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 00:07:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 00:07:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 00:07:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 00:07:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 00:07:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 00:07:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:07:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:07:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 00:07:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:07:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:07:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:07:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 00:07:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:07:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 00:07:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:07:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:07:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 00:07:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:07:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:07:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:07:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:07:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 00:07:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 00:07:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.88 s +[02-20 00:07:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:07:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:07:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 00:07:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/config.yaml +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 00:07:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the woman maintains her gentle smile, her eyes reflecting a sense of contentment and satisfaction. The warm, dim lighting continues to highlight her features, and the background remains unchanged, preserving the intimate and focused atmosphere of the entire video.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.json +[02-20 00:07:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.mp4 +[02-20 00:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 00:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 00:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 00:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 00:20:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 00:20:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:20:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:20:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:21:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 00:21:01|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..531904eefcb299eb6c7d72542e1b638a6ce14ab4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a close-up of the woman, showcasing her shoulder-length wavy brown hair and light-colored eyes. She starts with a warm, inviting smile, looking slightly upward as if engaged in a pleasant conversation. The soft, warm lighting highlights her natural makeup and the subtle texture of her hair, while the neutral background ensures the focus remains on her face.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0012e699425d4ddab692e1e71893abe38974323 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24c35d35be8c83be46b6cc583a55a137cccb5c14d5e2d1e33052d394892a9295 +size 466623 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..29ecf736348e8836dae7948735de0ffc98b32c5f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the woman's expression shifts to a more contemplative look. Her eyes narrow slightly, and her lips move gently, indicating she is listening intently or reflecting on what is being said. The warm lighting continues to cast a soft glow on her face, emphasizing the subtle changes in her expression.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51b9b284393ffd425196f2e0d560613fef1730e3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a8a0f0ea9ee1998e0bbbe0d5464a5a8d6644012e8040e0e7b015a827dd51523 +size 538144 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..03eebba530adeee123f63d52c5790dca957d2bcf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Her thoughtful demeanor persists as her eyes shift slightly, conveying a sense of engagement and attentiveness. The lighting remains consistent, maintaining the intimate and focused atmosphere of the shot. The background stays out of focus, ensuring all attention is on her facial expressions and subtle movements.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..db925070066e560699886ffa1017c56c0f16a3ad --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d5cebf026a5e1b53bcde054e9773f650d6e9202b9c9493b26bbbd0ea8ac8b30 +size 452099 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..050674b7fe9e335779eb179968a3598485078dac --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The woman's expression softens once more, transitioning into a gentle smile. Her eyes convey warmth and connection, suggesting a positive or reflective conclusion to her interaction. The camera remains steady, capturing the continuity of her appearance and the consistent lighting and background.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c32ba1048a3bdce4adb55d7d7be180da4b016ca9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fabb4b82941dad2aea1cce0b93e01259cd5e2ccb1f5136c8c105686ca3ee9c6a +size 421717 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..f4cf792bf863bdced401b1f221ba16bb4ec2415d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the woman maintains her gentle smile, her eyes reflecting a sense of contentment and satisfaction. The warm, dim lighting continues to highlight her features, and the background remains unchanged, preserving the intimate and focused atmosphere of the entire video.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9430203130c694da8b73a3338004bd0701bccdec --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c332cdb7bf66445c5b8d2da40f8b851c75fea328cb50972e9ad421b3b3598240 +size 419636 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a066869436e1d71b51bb172c824da24657e793e5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/console.log new file mode 100644 index 0000000000000000000000000000000000000000..2bd35398507e3c6438307e8ccb42375c0cc795ac --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/console.log @@ -0,0 +1,22 @@ +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:47:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:47:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/config.yaml +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.json +[02-20 13:48:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:01:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:01:23|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b843b6eab739e5e6a32a78f4ce36744d7316f1c7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/debug.log @@ -0,0 +1,353 @@ +[02-20 13:47:12|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 13:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 13:47:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 13:47:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 13:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 13:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 13:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 13:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 13:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 13:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 13:47:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 13:47:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 13:47:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 13:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 13:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 13:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 13:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 13:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 13:47:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 13:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 13:47:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 13:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 13:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 13:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:47:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:47:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:47:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 13:47:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:47:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 13:47:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:47:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:47:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 13:47:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:47:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:47:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 13:47:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 13:47:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.86 s +[02-20 13:47:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:47:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:47:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:48:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/config.yaml +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 13:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final moments of the video, the three men are nearing the bottom of the grand staircase. The man on the left, with his dark coat, light breeches, and white stockings, leads the way, followed by the middle man in his dark coat with a white cravat and light breeches. The man on the right, holding a small book or folder, completes the trio. The camera remains focused on the men, capturing their steady and deliberate movement down the stairs, with the luxurious interior elements providing a consistent and opulent setting throughout.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.json +[02-20 13:48:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:48:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.mp4 +[02-20 13:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 13:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 13:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 13:48:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 14:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 14:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:01:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:01:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 14:01:23|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d676d8d9fc0ce2d23c844b53d1e6d2bfd40c9941 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a wide shot of an opulent 18th-century grand staircase, featuring intricate wrought-iron railings and elaborate white plasterwork on the walls and ceiling. The red carpeting on the stairs adds a touch of luxury, and elegant candelabras and upholstered chairs with gold trim are visible at the base of the staircase. A single figure in period attire descends the stairs, followed by two more individuals joining from a higher landing, setting the stage for the main action.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2da5092a64271cca19f3eb10f83620414ca64dd5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:926f992ee4b53a7c5768aab7939611572bf160c17e3eeceae9130dfdf1022294 +size 1416522 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b9cbea41a97e64f6923dcc743fdc6fa03d39c690 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the previous scene, the three men dressed in period formal attire begin their descent down the grand staircase together. The man on the left wears a dark coat with light-colored breeches and white stockings, the middle man dons a similar dark coat with a white cravat and light breeches, and the man on the right wears a dark coat and holds a small book or folder. All three have powdered wigs typical of the era. The camera follows their movement, capturing the ornate details of the staircase and the luxurious surroundings.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..30e018db2ada203a345b30468007849da003d860 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4585dce747a6323403b02ec36a4425a6dbaa453bb54ecf2a4cb6305c7150b82 +size 1747242 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a802dcb05aebbc2bcb7935b65b5b6a1141268f50 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"As the men continue their descent, the camera maintains a steady follow shot, keeping them centered in the frame. The man on the left, with his dark coat, light breeches, and white stockings, walks confidently alongside the middle man, who is dressed in a dark coat with a white cravat and light breeches. The man on the right, holding a small book or folder, matches their pace. The ornate wrought-iron railings and the intricate plasterwork on the walls and ceiling remain prominent in the background, enhancing the grandeur of the setting.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8a6e0ce448dc7c44d8337af2f3357ff2c5ea62e7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f4a05bd9b46d5f281ad6c6fd455715aaf0e4573fc6f4bee00f51badefb724e6 +size 1563017 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..32e4f3191e7b6e0fe9ca2149e773eb5916cb2ff2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The men are now closer to the camera, continuing their descent down the grand staircase. The man on the left leads slightly ahead, while the middle man follows closely, and the man on the right completes the trio. Their powdered wigs and formal attire are consistently detailed, and their faces are clearly visible as they move. The ornate wrought-iron railings and the detailed plasterwork on the walls and ceiling continue to frame the scene, with the red carpeting, candelabras, and upholstered chairs at the base of the staircase providing a rich backdrop.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65b9bda9f8800766e8aa94c6d527455337f63542 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:952cdb2dda1b0f11d8f297738d2f1c6182cad8d7abfd71cfee518862ffd7e4da +size 1403209 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..53fa2cb3c71951f31c6b006d950167d8c4affaf3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments of the video, the three men are nearing the bottom of the grand staircase. The man on the left, with his dark coat, light breeches, and white stockings, leads the way, followed by the middle man in his dark coat with a white cravat and light breeches. The man on the right, holding a small book or folder, completes the trio. The camera remains focused on the men, capturing their steady and deliberate movement down the stairs, with the luxurious interior elements providing a consistent and opulent setting throughout.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6758d1aaacd2d45178c9837da007c2c255e1a77d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef92898c62d49b2f22a31079fc761625db93dcc7ac323bcd1e568a245a629de +size 1827396 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0cf3d8f28796780c913380dbcab355b3d024a1f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/console.log new file mode 100644 index 0000000000000000000000000000000000000000..8c835322fb748e8da50304aae469fd6dd9a713a6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/console.log @@ -0,0 +1,22 @@ +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 09:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 09:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/config.yaml +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.json +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:06:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:06:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:06:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:06:16|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..089f9ae6f5016596e283e07ed9b8f3902b663b46 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/debug.log @@ -0,0 +1,353 @@ +[02-20 09:52:09|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 09:52:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 09:52:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 09:52:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 09:52:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 09:52:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 09:52:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 09:52:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 09:52:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 09:52:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 09:52:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 09:52:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 09:52:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 09:52:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 09:52:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 09:52:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 09:52:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 09:52:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 09:52:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 09:52:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 09:52:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 09:52:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 09:52:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 09:52:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 09:52:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 09:52:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 09:52:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 09:52:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 09:52:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 09:52:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 09:52:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 09:52:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 09:52:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 09:52:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 09:52:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 09:52:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 09:52:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 09:52:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 09:52:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 09:52:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 09:52:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 09:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 09:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 09:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 09:52:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 09:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 09:52:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 09:52:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 09:52:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 09:52:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 09:52:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 09:52:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 09:52:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 09:52:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:52:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 09:52:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 09:52:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:52:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 09:52:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 09:52:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.57 s +[02-20 09:52:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 09:52:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:52:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/config.yaml +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 09:52:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The final segment showcases a wide shot of the ship's deck, with Captain Jack Sparrow standing prominently near the steering wheel. The crew is scattered across the deck, each member focused on their role, contributing to the sense of urgency and anticipation. The ship's intricate rigging and masts are fully visible, and the overall scene conveys a high-stakes maritime adventure." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.json +[02-20 09:52:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 09:52:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.mp4 +[02-20 09:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 09:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 09:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 09:52:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 10:06:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 10:06:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:06:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:06:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:06:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 10:06:16|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8536302e341bcdc581e4e30a75584bc4eaee58dd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with an intense close-up of Captain Jack Sparrow, focusing on his unique appearance: long, dreadlocked blonde hair embellished with beads and trinkets, a brown bandana wrapped around his forehead, a dark eye patch covering his right eye, and a well-groomed mustache and goatee. He is adorned with multiple layered necklaces featuring various pendants and wears a tattered brown pirate coat. The background hints at a ship's deck with blurred figures of other pirates.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14110c02b5d3806c8bb4efc056460f0adedb2881 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60afa2e5c2b60cfa4e72b697933dd5ee57adb05e181550af3fe0294262c347a0 +size 901057 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..95b1b8e923659cc5f6b99f2cdb4ad47f939e89c7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera begins to pull back, Captain Jack Sparrow turns slightly to his right and starts walking forward across the wooden deck. His dreadlocks flow behind him, and his layered necklaces glint in the light. The surrounding area becomes clearer, revealing other pirates in period attire, some holding weapons, and the ship's intricate rigging and masts in the background.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b3d83159f04b3e1bd07b3b5653acd8693e1a7e3c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6efde60303802ba463dde83bf061b0e97811d6eede02931187bca60c1311d6ee +size 1464874 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..398dbebc30cee88105bb8d32314929ce375a06ae --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"Captain Jack Sparrow continues his movement toward the ship's large wooden steering wheel, which dominates the foreground. The camera follows him closely, capturing the details of his tattered coat and the way his dreadlocks sway with each step. The bustling activity of the crew around him becomes more apparent, with some pirates climbing the rigging and others preparing for action.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ec81a7f1c92bf02d81fdb5843aecdd94a1f92b49 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf9259a66d928d81660124e3e3a242cae6b2dac047cdeab9ecbc6d6edcb8d43f +size 1932274 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..674d2b3afb16c81af931145fa54003ff811fa428 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The camera angle shifts upward, providing a broader view of the ship's deck. Captain Jack Sparrow is now positioned near the center, with his crew actively engaged in various tasks. Some pirates are climbing the masts, while others handle ropes and prepare for an imminent event. The ship's sails are partially unfurled, and the cloudy sky above adds to the dramatic atmosphere.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..725e8bd6f2a55907c6e0ec8ee58c7496d6a19d32 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8678ca8391f1e21b575626c6821239570f4e214e056f6411796dbd8b27747ea3 +size 1916404 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..4db49da7d5fb13337d8d2a5b5b0669a7d521b2c1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The final segment showcases a wide shot of the ship's deck, with Captain Jack Sparrow standing prominently near the steering wheel. The crew is scattered across the deck, each member focused on their role, contributing to the sense of urgency and anticipation. The ship's intricate rigging and masts are fully visible, and the overall scene conveys a high-stakes maritime adventure.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6a505344992110cf18e434350a66b61d3b06cd22 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ff7abebf49ae5fd9c9ee47961d5bd29ed0bc2904fd330f043b13007c8ce2f82 +size 1743712 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf10d30cd6f2653b720ae72b80b8c1ab1a71bf9c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/console.log new file mode 100644 index 0000000000000000000000000000000000000000..31b00b89c5e3f87e7f0ea72f17f337ce6bb3a05a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/console.log @@ -0,0 +1,22 @@ +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:59:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:59:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/config.yaml +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.json +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 15:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:13:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:13:10|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..acdd74ba701e887958b8bfdd185a1cb0f3698004 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/debug.log @@ -0,0 +1,353 @@ +[02-20 14:58:59|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 14:59:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:59:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:59:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 14:59:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 14:59:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 14:59:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 14:59:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 14:59:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 14:59:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 14:59:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 14:59:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 14:59:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 14:59:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 14:59:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 14:59:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 14:59:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 14:59:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 14:59:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 14:59:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 14:59:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 14:59:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 14:59:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 14:59:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 14:59:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 14:59:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 14:59:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 14:59:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 14:59:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 14:59:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 14:59:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 14:59:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 14:59:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 14:59:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 14:59:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 14:59:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 14:59:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:59:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:59:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 14:59:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:59:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:59:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:59:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 14:59:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:59:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 14:59:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:59:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:59:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 14:59:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:59:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 14:59:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 14:59:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 14:59:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 14:59:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:59:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:59:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:59:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:59:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 14:59:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 14:59:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.86 s +[02-20 14:59:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:59:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:59:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/config.yaml +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 14:59:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final moments, the women continue their warm embrace, with slight adjustments in their positions. The woman on the left, with her black and white fur coat, remains a central figure, while the woman in the middle, with her dark blue hat and grey scarf, shifts slightly. The woman on the right, in her beige trench coat, maintains her position, ensuring the group remains connected. The gentle snowfall and the building with stairs provide a serene and continuous backdrop, concluding the scene with a sense of warmth and connection.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.json +[02-20 14:59:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:59:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.mp4 +[02-20 14:59:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 14:59:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 14:59:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 14:59:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 15:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 15:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 15:13:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 15:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 15:13:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 15:13:10|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..7e4e1036c20ccf7c0addf5dbb24f81f93b63c8eb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with the three women standing close together in a snowy outdoor setting. The woman on the left, with long black hair, wears a black fur hat and a black and white fur coat. The woman in the middle, with short dark hair under a dark blue hat, is dressed in a light coat, grey scarf, and black gloves. The woman on the right, with a short dark bob, wears a beige trench coat and black gloves. They are positioned in front of a building with a staircase and informational posters, with snow gently falling around them.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d052b8f884821246904690d5275dc2aa2c1b5b46 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84295dce090510124fae199a1dad75a8e056a962a3ae5c74dc0fe1995f97f260 +size 1267366 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..20305e4dc719dcc71a5f34ffc276b92928b3fbc3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the women begin to embrace, with the woman on the left leaning into the center and the woman on the right wrapping her arms around both. The woman in the middle adjusts her stance slightly, accommodating the movement while maintaining her distinct appearance with the dark blue hat and grey scarf. The snowy environment and the building in the background remain consistent, providing a stable spatial context.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8801043a72e34ead55498d9b49cd95ead8b65334 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ba1a650183f712c0652f56676fc482ed1aa6341d9ababfa05576ecd5e6e8235 +size 1487253 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d2d0e6b9bd2e38db9b190af23f9b67a754ff29a7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The embrace continues, with the women shifting positions slightly. The woman on the left, in her black and white fur coat, becomes more prominent as she leans further into the group. The woman in the middle, with her dark blue hat and grey scarf, adjusts her posture, while the woman on the right, in her beige trench coat, keeps her arms wrapped around the group. The gentle snowfall and the building with stairs remain visible, enhancing the serene atmosphere.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab9bb112a6fde652443707a6df5417be44d2164d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a1ae0ca189753b6c35cc24a76e301e855a2477499a79354bd2bdade3410a296 +size 1297442 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9f69ac72eb908ce9455de150af0823aec7e4733c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The women maintain their close embrace, with subtle movements that highlight their individuality. The woman on the left, with her distinctive fur coat, remains a focal point, while the woman in the middle, with her dark blue hat and grey scarf, shifts slightly to accommodate the movement. The woman on the right, in her beige trench coat, ensures all three remain connected, with the snowy backdrop and the building providing a continuous and stable setting.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b32ca2b34e53951fe1b965cc2ce14a0973372bc7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:247ba780b1b2fe99a32df88b87de159806f157effda213774e6ea1adf65dcc14 +size 1368789 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..38c49bce8bbb6e1e154ef4722bd7f1f0fc54a5d9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments, the women continue their warm embrace, with slight adjustments in their positions. The woman on the left, with her black and white fur coat, remains a central figure, while the woman in the middle, with her dark blue hat and grey scarf, shifts slightly. The woman on the right, in her beige trench coat, maintains her position, ensuring the group remains connected. The gentle snowfall and the building with stairs provide a serene and continuous backdrop, concluding the scene with a sense of warmth and connection.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..626a851ba90b8929e4dc25c5d87edf36c6da3f13 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97247357210c9b16b6b09f241eaa191eb42ed857c3e6cbdf91f024c3ef95b076 +size 1160385 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..101e70f81f5856718a0266708d3bd5b4df1e194b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/console.log new file mode 100644 index 0000000000000000000000000000000000000000..6374de2d43b1852bebe015661402c709f6cfdcb7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/console.log @@ -0,0 +1,22 @@ +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:58:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:58:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:58:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:58:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/config.yaml +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.json +[02-20 03:58:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:12:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:12:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:12:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:12:15|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1efcd0be2338ebef2708a239b89659e4fd11f080 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/debug.log @@ -0,0 +1,353 @@ +[02-20 03:58:05|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 03:58:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:58:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 03:58:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 03:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 03:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 03:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 03:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 03:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 03:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 03:58:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 03:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 03:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 03:58:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 03:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 03:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 03:58:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 03:58:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 03:58:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 03:58:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 03:58:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 03:58:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 03:58:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 03:58:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 03:58:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 03:58:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 03:58:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 03:58:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:58:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 03:58:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 03:58:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:58:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:58:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:58:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 03:58:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:58:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 03:58:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:58:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:58:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 03:58:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:58:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 03:58:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 03:58:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 03:58:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 03:58:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:58:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 03:58:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:58:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:58:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 03:58:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 03:58:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.24 s +[02-20 03:58:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:58:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:58:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 03:58:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/config.yaml +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 03:58:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the man concludes his phone conversation, his expression reflecting the end of the dialogue. The white flip phone is still held to his ear, and his left hand, with the silver bracelet, remains visible. The background of green foliage persists, maintaining the serene outdoor atmosphere. The camera remains steady, capturing the man's final moments in the scene, ensuring identity and spatial consistency throughout the video." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.json +[02-20 03:58:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.mp4 +[02-20 03:58:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 03:58:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 03:58:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 03:58:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 04:12:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 04:12:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:12:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:12:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:12:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 04:12:15|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e276a6c3bca821b8784a179bc0dc0d895e0da89e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a close-up of the man, highlighting his dark, slightly messy hair and his attire: a yellow short-sleeved shirt layered under a brown sleeveless vest. He holds a white flip phone to his right ear, his left hand featuring a silver bracelet. Initially, he adjusts his hair with his right hand while listening to the call, his expression focused and attentive. The background is a soft blur of green foliage, suggesting an outdoor setting in natural daylight.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bcf68955188bda8a38d61d4861128700605fd3f9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ea30f89ffdb4305504f77dbf5a08cd9b9a4071e568e197faa43616f0999f476 +size 859741 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..8384e015966cde8682eda4ae64dce945102fb03a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the man continues his phone conversation, his posture and position remaining consistent. His facial expressions subtly shift, reflecting his engagement in the dialogue. The white flip phone stays firmly against his ear, and his left hand occasionally moves, possibly gesturing as he speaks. The background of green foliage remains unchanged, maintaining the serene outdoor ambiance.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..817c50fd55a47663a7b39818e877a5d72307c255 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ef2911a3a3da47fd5e5673664c2f5411b532fb11434d9b05b0b5853d85efc16 +size 948874 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..75b0be19c57e43f6538eb49a9598b4d45569cc52 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the next segment, the man's attention remains fixed on the conversation. His dark hair and casual clothing are still prominent, with the silver bracelet on his left wrist visible. The white flip phone is consistently held to his ear, and his slight head movements indicate active participation in the call. The background of blurred greenery continues to provide a tranquil setting, with no significant changes in the environment.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dab555dddfc2a2432138947fabbd4c44022aa354 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ce0d56d54e1d97139c9cec9b77a2d244fc90f5b70218ae51288505e63fa0874 +size 755477 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..df48006b522879634f975a6109fc6b43e7af820e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The video continues with the man still engaged in his phone call. His facial expressions and body language suggest a dynamic conversation, with occasional adjustments to his hair and phone placement. The background remains a consistent blur of green foliage, reinforcing the outdoor setting. The lighting highlights his features and attire, ensuring visual continuity.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c93e93793f4e78d7b53a0f790309ac41e8ad48f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030cced41604cbc561bdab763ea7828dbb7bcb710bd9c86b488fffe9e6dd2527 +size 936472 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..15e99180447d20dd2cc374d06b242b86455a69cd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the man concludes his phone conversation, his expression reflecting the end of the dialogue. The white flip phone is still held to his ear, and his left hand, with the silver bracelet, remains visible. The background of green foliage persists, maintaining the serene outdoor atmosphere. The camera remains steady, capturing the man's final moments in the scene, ensuring identity and spatial consistency throughout the video.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a5464a3d5e090b252b3db20f4e5f6147aca494e8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36dceab343f9206d021226372f9d573436c2d5e14f2aa7aa7ec99d732bbb38d6 +size 798720 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f462b29af2cbb9bf246e6498b819b4e610cc29ef --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ecf77840b2546071590e5276e79944eb192b7dfc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/console.log @@ -0,0 +1,22 @@ +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:05:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:05:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/config.yaml +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.json +[02-20 16:05:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:20:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:20:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:20:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:20:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8ee01f35c06793c152b333af03ea5a416f2f6365 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/debug.log @@ -0,0 +1,352 @@ +[02-20 16:05:00|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 16:05:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:05:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:05:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 16:05:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 16:05:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 16:05:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 16:05:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 16:05:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 16:05:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 16:05:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 16:05:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 16:05:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 16:05:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 16:05:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 16:05:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 16:05:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 16:05:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 16:05:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 16:05:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 16:05:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 16:05:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 16:05:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 16:05:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 16:05:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 16:05:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 16:05:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 16:05:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 16:05:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 16:05:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 16:05:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 16:05:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 16:05:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 16:05:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 16:05:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 16:05:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:05:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:05:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 16:05:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:05:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:05:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:05:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 16:05:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:05:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 16:05:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:05:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.05 s +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:05:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 16:05:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:05:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:05:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 16:05:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 16:05:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.65 s +[02-20 16:05:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:05:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:05:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 16:05:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/config.yaml +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 16:05:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with the boy in the center, wearing a light yellow polo shirt, turning slightly to his right and gesturing with his right hand towards the poster on the wall. The man to his left, dressed in a brown t-shirt and glasses, observes the boy with a focused expression. The girl to the boy's right, also in a light yellow polo shirt and a blue headband, stands quietly, her attention directed towards the poster. In the next segment, the boy continues to gesture, now pointing more directly at the girl, who remains still and attentive. The man shifts his gaze between the boy and the girl, showing interest in their interaction. The background, featuring a green potted plant, hanging bags, and an informational poster, remains unchanged throughout." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.json +[02-20 16:05:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 16:05:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 16:05:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 16:05:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 16:20:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 16:20:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:20:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:20:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:20:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 16:20:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f80aeba2ef551d2eb23f96d403621524812c074d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with the boy in the center, wearing a light yellow polo shirt, turning slightly to his right and gesturing with his right hand towards the poster on the wall. The man to his left, dressed in a brown t-shirt and glasses, observes the boy with a focused expression. The girl to the boy's right, also in a light yellow polo shirt and a blue headband, stands quietly, her attention directed towards the poster. In the next segment, the boy continues to gesture, now pointing more directly at the girl, who remains still and attentive. The man shifts his gaze between the boy and the girl, showing interest in their interaction. The background, featuring a green potted plant, hanging bags, and an informational poster, remains unchanged throughout.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0e076c88cba630a030ceaf90296397bb0ca39bb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fcff749539e572903b93453ef689781c256934b5e5e5602bc34d32df66d4b7e +size 1224089 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c34f9cce17b001df590233bb8c50d4f4009c493 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ee55ba499257b3240d1b3e2a3d869234cf5154fd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/console.log @@ -0,0 +1,22 @@ +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:26:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:26:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/config.yaml +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.json +[02-20 03:27:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:40:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:40:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:40:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:40:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e049eab82cb40074d5c87599f786dbd6bce7b4ce --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/debug.log @@ -0,0 +1,353 @@ +[02-20 03:26:44|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 03:26:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:26:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 03:26:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 03:26:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 03:26:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 03:26:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 03:26:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 03:26:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 03:26:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 03:26:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 03:26:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 03:26:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 03:26:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 03:26:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 03:26:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 03:26:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 03:26:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 03:26:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 03:26:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 03:26:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 03:26:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 03:26:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 03:26:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 03:26:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 03:26:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 03:26:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 03:26:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 03:26:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 03:26:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 03:26:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 03:26:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 03:26:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 03:26:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 03:26:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 03:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 03:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 03:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 03:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 03:26:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 03:26:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 03:26:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:26:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 03:26:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:26:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 03:26:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 03:26:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 03:26:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 03:26:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:26:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 03:26:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 03:26:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 03:26:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 03:26:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:26:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 03:27:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 03:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 03:27:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 03:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 03:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 03:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.65 s +[02-20 03:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 03:27:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 03:27:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 03:27:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/config.yaml +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 03:27:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the woman in the red dress maintains her position in the group, continuing her conversation with the man in the black tuxedo. The camera focuses on their interaction, capturing the details of their attire and the surrounding environment. The background elements, including the 'Frederic Zoller' poster and red flowers, remain prominent, reinforcing the opulent setting. Her short black hair and silver bracelet are consistently visible, ensuring identity persistence throughout the scene." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.json +[02-20 03:27:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.mp4 +[02-20 03:27:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 03:27:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 03:27:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 03:27:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 03:40:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 03:40:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 03:40:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 03:40:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 03:40:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 03:40:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..79174b4755fd48da51524842f52e98100742581a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with the woman in the red sleeveless dress and short black hair, her silver bracelet glinting as she gracefully descends the grand staircase with golden railings. She carries a beige handbag and is surrounded by elegantly dressed individuals, including men in military uniforms and women in glamorous attire. The background features a framed poster of 'Frederic Zoller' and vibrant red floral arrangements, setting the stage for an opulent event.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3d59b864772244e66a101ce5b18cae5444df30ad --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c8f873cec4a10f1d725bc7931ef7ca7cee64724976725e87a1ca4e8ae1d8345 +size 1894524 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..7e3362f542493e04f214c0a59e46e8db125e36e1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the woman reaches the bottom of the stairs, she turns to face forward, joining a group of formally dressed people. The camera follows her movement, capturing the details of her red dress and the interactions around her. The setting remains richly decorated with the 'Frederic Zoller' poster and red flowers, enhancing the sophisticated atmosphere. She engages in conversation with a man in a white military uniform and another in a black tuxedo, both holding glasses of champagne.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ebb0841f558dab7004720a6404a4d40b2366b870 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f95e7ada90d1bca92dcc9543cb8910ca40df4f288a8de9cdb9070b67f22a0fa +size 1752715 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..bf4c0e62d87d052ce9e20ae73af885878cf9b518 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The woman in the red dress continues her conversation with the man in the black tuxedo, their interaction lively and animated. The camera focuses on their exchange, highlighting the details of their attire and the surrounding environment. The background elements, including the 'Frederic Zoller' poster and red flowers, remain prominent, reinforcing the opulent setting. Her short black hair and silver bracelet are consistently visible, ensuring identity persistence.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..faa261957f7fdcef0e18c1d4c875e192e33a028f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:779f01071b782988beb406f834fe7807a91a5da44d07132933308dc3498fb259 +size 1402604 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..36567bd573510eafd6fa51e64d0390e45e1e4b05 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The scene transitions smoothly as the woman in the red dress turns slightly, her posture and expression indicating an engaging conversation with the man in the tuxedo. The camera captures the details of their interaction, with the background elements remaining consistent. The 'Frederic Zoller' poster and red flowers continue to enhance the sophisticated ambiance, while the woman's red dress and silver bracelet are clearly visible.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8969c32aba433ab97256219779d99189f1fa8273 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92e30f83d218270f6fe7b19941dcf6ee62ba91a4de4daf0ccc52b196a67e756c +size 1307706 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c34f069b86d29f336a920ae2f0502543745ff3eb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the woman in the red dress maintains her position in the group, continuing her conversation with the man in the black tuxedo. The camera focuses on their interaction, capturing the details of their attire and the surrounding environment. The background elements, including the 'Frederic Zoller' poster and red flowers, remain prominent, reinforcing the opulent setting. Her short black hair and silver bracelet are consistently visible, ensuring identity persistence throughout the scene.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6804048c59971aaaaca07c119bc7e96a634d4d5b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:407073386abce1ef7fcb249aa6c86141d65310db7cca0cd35e55bf3f1875098f +size 935404 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..797e5d021c67347fe72e852b97ac48141a8a7543 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/console.log new file mode 100644 index 0000000000000000000000000000000000000000..6efd8ec6b17e6044f2c77eb3b438a71cd17245af --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/console.log @@ -0,0 +1,22 @@ +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:29:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:29:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/config.yaml +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.json +[02-20 07:30:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:43:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:43:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:43:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:43:27|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a9ccc63e631aef7d680cc25a0c844a360525fd0e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/debug.log @@ -0,0 +1,353 @@ +[02-20 07:29:20|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 07:29:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:29:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 07:29:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 07:29:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 07:29:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 07:29:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 07:29:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 07:29:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 07:29:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 07:29:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 07:29:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 07:29:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 07:29:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 07:29:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 07:29:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 07:29:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 07:29:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 07:29:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 07:29:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 07:29:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 07:29:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 07:29:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 07:29:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 07:29:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 07:29:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 07:29:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 07:29:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 07:29:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 07:29:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 07:29:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 07:29:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 07:29:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 07:29:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 07:29:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 07:29:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 07:29:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 07:29:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 07:29:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 07:29:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 07:29:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 07:29:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:29:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 07:29:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:29:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 07:29:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 07:29:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 07:29:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 07:29:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:29:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 07:29:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 07:29:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 07:29:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 07:29:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 07:29:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 07:29:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 07:29:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 07:30:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 07:30:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 07:30:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.20 s +[02-20 07:30:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 07:30:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 07:30:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/config.yaml +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 07:30:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final moments of the video, the man and woman continue their walk through the elegantly decorated room. The man, still wearing his dark jacket and collared shirt, turns slightly towards the woman as they converse. The woman, in her light pink sweater, smiles and responds to him, her expression indicating engagement in the conversation. The ornate white metal gate is still visible in the foreground, but the background now shows more of the room's details, including a brick-patterned section of the wall and a black railing. The camera maintains a steady follow shot, keeping both individuals in focus as they move through the space." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.json +[02-20 07:30:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 07:30:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.mp4 +[02-20 07:30:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 07:30:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 07:30:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 07:30:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 07:43:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 07:43:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 07:43:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 07:43:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 07:43:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 07:43:27|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f1a145b92b79dcaf2408a299babcb0b33702a4ba --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a view through an ornate white metal gate, which frames the entrance to a warmly lit interior. A man with short dark hair and a gray beard, wearing a dark jacket over a collared shirt with white trim, steps forward from behind the gate. His face becomes fully visible as he begins to speak, gesturing with his hands. The background features a wooden door with glass panes and decorative wall patterns, adding to the elegant ambiance of the setting.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4e7864f65c9cdc88cfd1de31d9d4c38efcda64f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0333a4c11c5abfca651d24dd68a8e190a8e7c82337ece6a99dd12e29b5a77e91 +size 847814 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5a63996accb5af547e871767b35eb905d4458769 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the man continues to speak, a woman with short dark hair, dressed in a light pink ribbed sweater, enters from behind him. She walks alongside him, her attention focused on him as he gestures and explains something. The ornate white metal gate remains in the foreground, framing their movement as they proceed deeper into the room. The background reveals more of the interior, including a glimpse of a railing and additional decorative elements on the walls.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bfaf7e9fa6ec6eca4b824c31f31c1691c58f12e1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43d494d10e28ee9d64b9770b64edb471f3389967dd5597c54c4c2e90fccb8eea +size 1166911 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0d47efede9baee402e3982f1200378be55e60e40 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The man and woman continue their walk through the elegantly decorated room, moving closer to the camera. The man, still wearing his dark jacket and collared shirt, turns slightly towards the woman as they converse. The woman, in her light pink sweater, smiles and responds to him, her expression indicating engagement in the conversation. The ornate white metal gate is still visible in the foreground, but the background now shows more of the room's details, including a brick-patterned section of the wall and a black railing.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e5a40ef13b54e85386a4147781aca7ad3bb85544 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4e0c4df06ed15565fbcbb72f4af07083d5321036a9a1fc950f0c2fdae6d6410 +size 1148736 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..62eb4653055f92ca810ec93f9dc860989292e818 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The couple moves further into the room, their interaction becoming more animated. The man gestures with his hands as he speaks, emphasizing his points, while the woman listens attentively, nodding and smiling. The ornate white metal gate remains in the foreground, but the background reveals more of the room's intricate details, including a black railing and additional decorative elements on the walls. The camera follows them closely, capturing their dynamic interaction.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a433b9ce23fcf374cf6cd7051c2ec305ea9bffab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eaf78b9119b02924a58da917e95af72f220afd0decf94df1702d446e095ff56 +size 1330426 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..fca28e9b149c68f13f7797fe2b96d4832418217a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments of the video, the man and woman continue their walk through the elegantly decorated room. The man, still wearing his dark jacket and collared shirt, turns slightly towards the woman as they converse. The woman, in her light pink sweater, smiles and responds to him, her expression indicating engagement in the conversation. The ornate white metal gate is still visible in the foreground, but the background now shows more of the room's details, including a brick-patterned section of the wall and a black railing. The camera maintains a steady follow shot, keeping both individuals in focus as they move through the space.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bfbdc1f0f1b390543147887a030e47d95541b27a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16d81bc121ba83649dbebe2794cc6d6c7de2817d293081a2db989af5915ccaad +size 1214853 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e33d68e2394d7b9e759cc1b5e93a0286f9628e8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c95190f0caf9ee0c7d3b6e72fe1e333721fea919 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/console.log @@ -0,0 +1,22 @@ +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 20:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 20:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/config.yaml +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.json +[02-20 20:41:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 20:54:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 20:54:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 20:54:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 20:54:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9342a89135c216231d6a1c27b4d6897172b33d40 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/debug.log @@ -0,0 +1,352 @@ +[02-20 20:40:44|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 20:40:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 20:40:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 20:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 20:40:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 20:40:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 20:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 20:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 20:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 20:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 20:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 20:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 20:40:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 20:40:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 20:40:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 20:40:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 20:40:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 20:40:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 20:40:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 20:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 20:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 20:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 20:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 20:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 20:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 20:40:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 20:40:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 20:40:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 20:40:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 20:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 20:40:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 20:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 20:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 20:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 20:40:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 20:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 20:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 20:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 20:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 20:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 20:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 20:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 20:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 20:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 20:40:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 20:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 20:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 20:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 20:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 20:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.21 s +[02-20 20:41:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 20:41:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 20:41:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 20:41:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/config.yaml +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 20:41:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="At the start of the video, the man is seen walking across a sloped roof, captured from behind by a following camera. He has blonde hair, wears sunglasses, a yellow Hawaiian shirt with a floral design over a white 'CHAMPION' t-shirt, blue jeans, and a brown leather tool belt. The background features dense green foliage, and the bright sunlight highlights his relaxed demeanor. As he progresses, he reaches up to adjust his hair, continuing his steady walk. In the final portion, he removes his open Hawaiian shirt, revealing his bare torso while still wearing his sunglasses and tool belt, maintaining his confident stride across the roof." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.json +[02-20 20:41:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 20:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 20:41:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 20:41:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 20:54:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 20:54:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 20:54:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 20:54:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 20:54:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 20:54:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c0205a7b52dba8eab49960ed5f25a85c76aa90c9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"At the start of the video, the man is seen walking across a sloped roof, captured from behind by a following camera. He has blonde hair, wears sunglasses, a yellow Hawaiian shirt with a floral design over a white 'CHAMPION' t-shirt, blue jeans, and a brown leather tool belt. The background features dense green foliage, and the bright sunlight highlights his relaxed demeanor. As he progresses, he reaches up to adjust his hair, continuing his steady walk. In the final portion, he removes his open Hawaiian shirt, revealing his bare torso while still wearing his sunglasses and tool belt, maintaining his confident stride across the roof.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..80fb9eb9c8333be6968eaf8bdcd8a191c0f5163a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4202db57001903ba53df5d1d116fb9c9ec4270f94b2b3a0983c5cb0d83b2a62 +size 1974200 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8dc12dbcc7bf60111b99c4d7fb76c9211bb8c26 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/console.log new file mode 100644 index 0000000000000000000000000000000000000000..479822e06cf9ee8476ed8653335cb89be2275164 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/console.log @@ -0,0 +1,22 @@ +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/config.yaml +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.json +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:02:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:02:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:02:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:02:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..64a14208e4af8191bdc1f30322c1d6b1ff11a4e5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/debug.log @@ -0,0 +1,353 @@ +[02-19 20:47:50|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 20:47:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 20:47:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 20:47:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 20:47:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 20:47:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 20:47:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 20:47:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 20:47:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 20:47:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 20:48:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 20:48:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 20:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 20:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 20:48:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 20:48:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 20:48:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 20:48:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 20:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 20:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 20:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 20:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 20:48:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 20:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 20:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 20:48:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:48:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 20:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:48:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 20:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:48:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 20:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-19 20:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 20:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 20:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 20:48:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:48:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 20:48:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 20:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 20:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:48:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 20:48:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 20:48:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.78 s +[02-19 20:48:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:48:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/config.yaml +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:48:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the video concludes with a lingering shot of the couple in their embrace. The man's head is slightly turned, his expression reflecting a mix of serenity and contemplation. The woman keeps her eyes closed, her face relaxed and content. The camera slowly pulls back, revealing more of the room's details, including the ornate decorations and the steady flame of the candles. The scene ends with the couple still holding each other, their identities and positions consistent throughout, leaving a lasting impression of their tender connection." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.json +[02-19 20:48:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:48:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.mp4 +[02-19 20:48:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 20:48:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 20:48:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 20:48:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 21:02:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 21:02:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:02:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:02:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:02:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 21:02:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e4d6caf5d86639a1949203fa926c4dac0d41018d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a close-up shot of an elderly couple in a dimly lit, traditional East Asian room. The man, with a long gray beard and hair tied back, wears a light-colored robe with gold embroidery. The woman, with long gray braided hair, is dressed in a dark black robe with intricate patterns and a golden headpiece. They are positioned near a wooden lattice window, with flickering candlelight illuminating their faces. The man gently places his hand on the woman's shoulder as she leans into him, their expressions conveying a sense of deep connection.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e39c0060d8a47cba1ec804145bc7273470b447d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e178b17b5a2139644cd703bcb187bbf036ce4ce12fcf7b994cd9bf174155854f +size 978419 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..daf2cf5500ce45535965b547282ef0768548f27c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the previous scene, the couple remains in their embrace. The woman rests her head on the man's shoulder, her eyes closed in a state of peaceful contentment. The man's expression is calm and composed, his gaze directed slightly downward. The camera slowly pans around them, capturing the subtle movements of their heads and the gentle sway of their bodies. The background remains consistent, with the wooden lattice window and the flickering candles providing a warm, ambient light.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3d4ddd7594b48337b708dba1a93a4fdc082899be --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd3d6e82afaf37f972f5c69e218de4fd3ac6402b0e31c1f3f02e1de6ae1f05f4 +size 940825 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..39206eccd8ddd821b8c706793d3838a73680d4d5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"As the scene progresses, the man slightly turns his head, his expression softening further as he looks at the woman. The woman keeps her eyes closed, her face relaxed and serene. The camera maintains its focus on their faces, highlighting the intricate details of their clothing and the ornate decorations on the woman's headpiece. The background elements, including the wooden lattice and the candles, remain unchanged, reinforcing the intimate and timeless atmosphere of the moment.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..877632938f796767e7d121708c084a6b144ef1d5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f68c39a62f8318473cc4781f7327a2f644c54e44ca9c7de6ddf05ca2bc024cb +size 800061 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ff9c4138614f543d959d6420cc1ce5f8c9502406 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the next segment, the couple continues to stand close together, their embrace unwavering. The man's hand remains gently placed on the woman's shoulder, and her head stays resting on his shoulder. The camera subtly shifts, capturing the slight changes in their facial expressions and the gentle movement of their hair. The warm candlelight continues to cast a soft glow on their faces, enhancing the sense of tranquility and intimacy.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5220358ead28988feffde74622b1355e00d9f86c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30b5b69735fdbfa15222ea8cb9cd150937e056d8f35167563c1e3057bce6bff6 +size 837815 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..0b83afcd5b9d402ccbb6f305dd455f78e2b70982 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the video concludes with a lingering shot of the couple in their embrace. The man's head is slightly turned, his expression reflecting a mix of serenity and contemplation. The woman keeps her eyes closed, her face relaxed and content. The camera slowly pulls back, revealing more of the room's details, including the ornate decorations and the steady flame of the candles. The scene ends with the couple still holding each other, their identities and positions consistent throughout, leaving a lasting impression of their tender connection.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..86522ff60bacd77fcbc93e92705e3bf96a59532a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6974b6f08c61e5af54457cd455f3f937e7a86d090a0f8bf169155a8860ce13e1 +size 795460 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6bf3877de91b484118a8e21d385ac92c7e3a8ee --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/console.log new file mode 100644 index 0000000000000000000000000000000000000000..dfb0825865797d85912fd4003bc284656ddcee5c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/console.log @@ -0,0 +1,22 @@ +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:17:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:17:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/config.yaml +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.json +[02-20 14:18:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:32:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:32:37|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..00b211c1ebc935972a7eb815804626a3d6e2e425 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/debug.log @@ -0,0 +1,353 @@ +[02-20 14:17:24|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 14:17:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:17:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 14:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 14:17:33|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 14:17:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 14:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 14:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 14:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 14:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 14:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 14:17:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 14:17:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 14:17:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 14:17:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 14:17:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 14:17:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 14:17:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 14:17:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 14:17:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 14:17:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 14:17:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 14:17:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 14:17:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 14:17:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 14:17:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 14:17:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 14:17:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 14:17:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 14:17:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 14:17:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 14:17:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:17:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 14:17:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:17:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 14:17:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 14:17:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 14:17:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 14:17:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 14:17:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 14:17:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 14:17:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 14:18:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 14:18:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 14:18:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.63 s +[02-20 14:18:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 14:18:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 14:18:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 14:18:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/config.yaml +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 14:18:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the woman maintains her focus on the conversation, her posture steady and her expression thoughtful. The beige handbag remains on her lap, and the overall ambiance of the room stays warm and subdued, emphasizing the personal nature of the interaction.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.json +[02-20 14:18:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.mp4 +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 14:18:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 14:18:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 14:18:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 14:32:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 14:32:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 14:32:37|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0075ecbf9bb1365ddaf746464e2143e7485f9eeb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up shot of a woman seated in a dimly lit room, her face softly illuminated by ambient light. She has medium-length dark hair pulled back, revealing her delicate features. She is wearing a light-colored floral short-sleeve dress with a subtle pattern, a thin gold necklace with a small pendant, and a ring on her left ring finger. A beige handbag rests on her lap. Initially, she appears contemplative, looking down at the handbag.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca749b2e6922dc77d73577f5932c8b106fa8c922 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adb366cbdec11f3f2be9aa5f9a169ba62b8bffcf5c8707c6f2881270787f07e1 +size 406792 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2da72b790980cb6e374f88ddefa8ab1ace0ee5d2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the previous scene, the woman reaches into the beige handbag resting on her lap and retrieves a smartphone. She holds the phone in her right hand, examining it briefly before preparing to make a call.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ddae7b316f2e031749b3c6093a017ce60af60063 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:090482925e8f7f7f41752ab90679f0406252d1bef2cd5e5922389e69e0e01011 +size 701016 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e2155fbe5652c2192007a327a9fb44bcd245488f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The woman brings the smartphone to her ear, beginning a conversation. Her expression shifts subtly as she listens and speaks, indicating attentiveness and involvement in the discussion. The background remains blurred, maintaining the intimate atmosphere.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..81c25dbe919ba4251a625b72dcc5fa2c10147a54 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07469aa052d8242ebcd61861782a74607e15901a740d11aa9c32989a2e2c6615 +size 612259 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b458b5d08cd924b4727472526c7ddfe309c0b6d7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the conversation continues, her facial expressions change slightly, reflecting her engagement and emotional response to the dialogue. The lighting remains consistent, highlighting her features and the details of her attire.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ed3e41333ef25e47264c029be8e3ccda12605066 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aa453c35e99e3b0b5bb984238cbd297363e9b609efb0b29dc2f6ea48ec0afe0 +size 733701 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..7167564409ce558ab8536da473db0a4be2bdd0d9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the woman maintains her focus on the conversation, her posture steady and her expression thoughtful. The beige handbag remains on her lap, and the overall ambiance of the room stays warm and subdued, emphasizing the personal nature of the interaction.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29d5ad976d19edd7bd2d57bac3e8ea4a419a401e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a99a682e00fdf22068f4f2cbfaca188ceb9c5a01be566fde1f09e9d703114e26 +size 590364 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf234901ebdf87df7814ad5ff51dd83c0641c605 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b96ce719f49e5a74fb254ffee30d7697e751a9fc --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/console.log @@ -0,0 +1,22 @@ +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-21 00:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-21 00:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-21 00:09:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-21 00:09:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/config.yaml +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.json +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-21 00:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-21 00:23:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-21 00:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-21 00:23:07|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..38263218fbbad03bd8d2116de469d424d234359a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/debug.log @@ -0,0 +1,353 @@ +[02-21 00:08:56|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-21 00:09:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-21 00:09:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-21 00:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-21 00:09:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-21 00:09:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-21 00:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-21 00:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-21 00:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-21 00:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-21 00:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-21 00:09:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-21 00:09:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-21 00:09:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-21 00:09:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-21 00:09:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-21 00:09:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-21 00:09:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-21 00:09:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-21 00:09:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-21 00:09:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-21 00:09:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-21 00:09:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-21 00:09:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-21 00:09:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-21 00:09:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-21 00:09:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-21 00:09:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-21 00:09:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-21 00:09:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-21 00:09:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-21 00:09:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-21 00:09:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-21 00:09:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-21 00:09:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-21 00:09:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-21 00:09:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-21 00:09:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-21 00:09:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-21 00:09:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-21 00:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-21 00:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-21 00:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-21 00:09:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-21 00:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-21 00:09:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-21 00:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-21 00:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-21 00:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-21 00:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-21 00:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-21 00:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-21 00:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-21 00:09:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-21 00:09:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-21 00:09:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-21 00:09:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-21 00:09:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-21 00:09:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-21 00:09:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.97 s +[02-21 00:09:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-21 00:09:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:09:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/config.yaml +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-21 00:09:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the man in the blue military-style uniform concludes his speech, his expression remaining animated. His round gold-rimmed sunglasses and white gloves are still prominent, and the colorful medals on his chest add to his distinguished look. The two women flanking him remain in their respective positions; the woman on the left in the striped dress with pearl accessories and the woman on the right in the purple lace dress. The floral folding screen in the background provides a stable and elegant setting. The camera maintains its focus on the man, capturing his final words as the video ends.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.json +[02-21 00:09:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-21 00:09:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.mp4 +[02-21 00:09:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-21 00:09:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-21 00:09:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-21 00:09:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-21 00:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-21 00:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-21 00:23:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-21 00:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-21 00:23:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-21 00:23:07|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..762466b1e751040ffac4e44e5eab99d94bc3d915 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a close-up shot of a man in a blue military-style uniform, complete with white lace collar and epaulets, and multiple colorful medals on his chest. He is bald, with a prominent mustache, and wears round gold-rimmed sunglasses. His white-gloved hand is visible as he starts speaking. To his left stands a woman in a beige and white striped dress with a high collar, pearl necklace, and green trim, while to his right is a woman in a light purple lace dress. The background features a traditional folding screen with floral patterns. The camera is steady, focusing on the man as he begins to speak.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1ec3c12145df78c3af020d4b515d28d37a602fd9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c84fb8efde7dee39e4957d2fb3ce858e905d36d9b857c01aa1c69d78e7c95673 +size 825639 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..446f883da3b633af19ab140152c317e412dbb429 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Continuing from the initial scene, the man in the blue military-style uniform remains the central figure, speaking animatedly. His round gold-rimmed sunglasses and white gloves are still prominent, and he makes slight gestures with his gloved hand. The two women beside him remain in their respective positions; the woman on the left in the striped dress with pearl accessories and the woman on the right in the purple lace dress. The floral-patterned folding screen in the background remains unchanged, providing a consistent and elegant setting. The camera angle remains fixed, keeping the man centered.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ad9618f79b83f9f2f90c7de2f5ffb4f301a5e51 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69d58c0ca81aa4070e52d6d10aeb6042cc0f61f3aad0615a7959c3a8a2fd018a +size 874265 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..09c48e68f6652c098361cfb509879f88fc047889 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"As the video progresses, the man in the blue military-style uniform continues his speech, his expression becoming more animated. His round gold-rimmed sunglasses and white gloves are still visible, and the colorful medals on his chest add to his distinguished appearance. The two women flanking him remain in place, the woman on the left in her striped dress with pearl accents and the woman on the right in her purple lace dress. The floral folding screen in the background provides a stable and elegant backdrop. The camera maintains its focus on the man, capturing his continued dialogue.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..616935bb5b1259cb488595339d9902ffb5658d91 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a0f47589cdcfbaa3659178a87f966b45bdd86512309d0565b944d8a9a953f18 +size 786032 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..151eb5b4a658460a6216fecedb8fbdbf92e930c2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The scene transitions smoothly as the man in the blue military-style uniform continues to speak, his gestures becoming more pronounced. His round gold-rimmed sunglasses and white gloves remain a focal point, and the colorful medals on his chest are clearly visible. The two women beside him remain stationary; the woman on the left in the striped dress with pearl accessories and the woman on the right in the purple lace dress. The floral-patterned folding screen in the background remains a constant element, providing a consistent setting. The camera angle stays fixed, ensuring the man remains the primary focus.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65dc90d074bb3fb40eaab0b197e6c96232fad1a7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dfa03f7494a6b6b83ff5537fc5a52cfd6bb68e6b7c7b99d253c46ac66b2aa8b +size 1011014 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..fb5b578cec77d5e8b9c6b3f890bf8f7a0e741fcb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the man in the blue military-style uniform concludes his speech, his expression remaining animated. His round gold-rimmed sunglasses and white gloves are still prominent, and the colorful medals on his chest add to his distinguished look. The two women flanking him remain in their respective positions; the woman on the left in the striped dress with pearl accessories and the woman on the right in the purple lace dress. The floral folding screen in the background provides a stable and elegant setting. The camera maintains its focus on the man, capturing his final words as the video ends.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..759b031858fba059832ecfd94ce96452423731db --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36abf334d401d59deeb8fa631e927a1b1b7f4b261f79bf2b1858c55bf5ad8b94 +size 795650 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe49e56f0206b408c71fb49412808a4413abca76 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ed9562f75caef2018563306d496400ed018d22ff --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/console.log @@ -0,0 +1,22 @@ +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:01:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:01:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/config.yaml +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.json +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:15:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:15:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ce4b5ffc10ccc313a652cd0c4a8f23668eb49bec --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/debug.log @@ -0,0 +1,353 @@ +[02-20 10:01:05|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 10:01:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:01:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 10:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 10:01:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 10:01:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 10:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 10:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 10:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 10:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 10:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 10:01:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 10:01:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 10:01:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 10:01:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 10:01:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 10:01:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 10:01:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 10:01:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 10:01:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 10:01:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 10:01:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 10:01:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 10:01:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 10:01:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 10:01:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 10:01:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 10:01:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 10:01:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 10:01:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 10:01:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 10:01:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 10:01:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 10:01:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 10:01:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 10:01:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 10:01:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 10:01:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 10:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 10:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 10:01:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 10:01:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 10:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 10:01:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 10:01:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:01:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 10:01:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:01:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 10:01:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 10:01:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 10:01:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 10:01:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:01:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 10:01:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 10:01:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 10:01:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 10:01:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 10:01:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 10:01:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 10:01:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 10:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 10:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 10:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.76 s +[02-20 10:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 10:01:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 10:01:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/config.yaml +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 10:01:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='The scene concludes with the group still gathered in the hallway. The woman in the orange and black striped jumpsuit faces the camera directly, continuing her conversation with the man in the pink suit. The woman in the leopard print top and white pants stands close by, while the woman in the black lace dress and the man in the purple outfit are positioned further back, observing the interaction. The camera remains steady, capturing the formal and composed demeanor of all individuals, with the background elements remaining consistent throughout.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.json +[02-20 10:01:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:01:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.mp4 +[02-20 10:01:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 10:01:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 10:01:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 10:01:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 10:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 10:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:15:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:15:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:15:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 10:15:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..76c09af2957a38066721cf5df33577e510b4657b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a group of elegantly dressed individuals walking down a bright, modern hallway. The camera focuses on a woman wearing an orange and black vertically striped off-shoulder jumpsuit, black high-heeled sandals, and large hoop earrings. She walks away from the camera, then turns around to greet a man in a pink suit. The hallway is adorned with tall black vases and features a well-lit interior space with shelves displaying various items in the background.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41badd396582d40f959642cd54641beaf1ef2ba2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6ed62b5f5a5deac92c634ad4f9fb3b0ce25ed65fb317eded5c47a157894c624 +size 1298969 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b25f2d8055506c1744da0b879073fe46a47a8226 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the woman in the orange and black striped jumpsuit shakes hands with the man in the pink suit. Another woman, dressed in a leopard print top and white pants, stands beside them, observing the interaction. The camera follows their movement, capturing the formal and composed demeanor of the group. In the background, other individuals begin to appear, including a woman in a black lace dress holding a white folder.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57907f275686e4741a8c601356be1c54cbdaffe4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:728c7223b3de81f78ab93bbd3f9e3dd2b841891ab93d2bb93685dddba5fd3a86 +size 2396142 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4c2dc9e770c8ba8da787def7df55af56e86c4e1a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The group continues to gather in the hallway, with the woman in the orange and black striped jumpsuit engaging in conversation with the man in the pink suit. The woman in the leopard print top and white pants remains by their side, while the woman in the black lace dress and the man in the purple outfit are now more prominently visible in the background. The camera angle shifts slightly to include these new arrivals, maintaining the formal and elegant atmosphere of the scene.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24d5912063a10d493a41b04e80e480eca755dffa --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6dff6c840eb762d317a28efc1ff1ddf5b782f2f1d5ec029220a92b8136842b8 +size 2086824 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a951b65b7e62e30db9521b45bcf7f8ea1f68d775 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The interaction between the woman in the orange and black striped jumpsuit and the man in the pink suit continues, with the woman gesturing as they converse. The woman in the leopard print top and white pants stands close by, while the woman in the black lace dress and the man in the purple outfit observe the exchange. The background remains consistent, with the tall black vases and well-lit interior space providing a sophisticated setting.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6327e8da85408755c085d19adcdf8114ca910287 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5768d99fa17f0b59f720eaadb8fb4b73726c60bd784ca75839eba60a54a5fb3e +size 1874008 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..9157d6cf978207b3037bad7b72d91bd9965eed1e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The scene concludes with the group still gathered in the hallway. The woman in the orange and black striped jumpsuit faces the camera directly, continuing her conversation with the man in the pink suit. The woman in the leopard print top and white pants stands close by, while the woman in the black lace dress and the man in the purple outfit are positioned further back, observing the interaction. The camera remains steady, capturing the formal and composed demeanor of all individuals, with the background elements remaining consistent throughout.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e4726e04d7135a173fe1f6cc502b35428522647 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89185b60c553eeee6de0316fbabda1c60cec899dd7371058fe899390a7b5c710 +size 1346250 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc9c007fbedd49944fdc711ba6cc1df7434e22be --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/console.log new file mode 100644 index 0000000000000000000000000000000000000000..3ea7e51dcb2b4cd1f7e273765c03cf1fb9b4e731 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/console.log @@ -0,0 +1,22 @@ +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:59:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:59:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/config.yaml +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.json +[02-19 22:00:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:13:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:13:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:13:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:13:31|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..645858cec5da4e55bdfc7abb6dbc61f9c9c08fcb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/debug.log @@ -0,0 +1,353 @@ +[02-19 21:59:16|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 21:59:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:59:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 21:59:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 21:59:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 21:59:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 21:59:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 21:59:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 21:59:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 21:59:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 21:59:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 21:59:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 21:59:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 21:59:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 21:59:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 21:59:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 21:59:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 21:59:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 21:59:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 21:59:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 21:59:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 21:59:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 21:59:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 21:59:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 21:59:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 21:59:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 21:59:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 21:59:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 21:59:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 21:59:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 21:59:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 21:59:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 21:59:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:59:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 21:59:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 21:59:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 21:59:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 21:59:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 21:59:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 21:59:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.77 s +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 21:59:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 21:59:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.61 s +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 1.15 s +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 1.15 s +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 1.15 s +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 21:59:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 21:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 21:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 21:59:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 22:00:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 22:00:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 22:00:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 30.98 s +[02-19 22:00:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 22:00:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 22:00:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/config.yaml +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 22:00:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the woman places the glass back on the table, her hands clasped together as she continues to speak. The man listens, his posture relaxed but attentive. The camera maintains a steady focus on their interaction, capturing the nuances of their conversation. The room's vintage charm, with its floral bedding, side table lamp, and shelves filled with decorative items, provides a consistent backdrop throughout the scene." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.json +[02-19 22:00:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 22:00:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.mp4 +[02-19 22:00:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 22:00:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 22:00:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 22:00:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 22:13:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 22:13:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 22:13:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 22:13:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 22:13:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 22:13:31|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c64d84e1b3d7eb3888a6cb18f65430a7d291530e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a close-up of the woman, her dark brown hair neatly pulled back and her white blouse with black dot patterns and blue patterned sleeves clearly visible. She appears to be speaking, her expression conveying a sense of earnestness. The man, partially visible on the left, is seated across from her, listening intently. The background hints at a vintage room with dark curtains and a shelf holding various items.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f79a9c9ece3bb990dfad46a3c6f88f6f3e251dc8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01ef9e9f36d89ffb77a14bd2d71aedc72fe91d63c391da7551c780c7afb9bc70 +size 525234 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..bfceb974eda4b85a1740a80c997a03d8ae007930 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera slowly zooms out, the full view of the man becomes apparent. He is wearing a light-colored shirt and has his arms crossed on the wooden table. The woman continues to speak, gesturing with her hands to emphasize her points. The room's details become more evident, including a bed with floral bedding, a side table with a lamp, and shelves displaying decorative objects. The interaction between the two remains the focal point, with the woman leading the conversation.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e5051e41224d508a86c3d189229c89f45298109f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:416cf03a3f9be0e1f3b5fc858de27c347a60158faa533aba6b7e24edbb2e3de4 +size 1130227 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..fad68825bfc014819976e1e113749c77c3349f66 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera angle shifts slightly, providing a clearer view of the woman's face and upper body. She continues to engage in the conversation, her expressions and gestures indicating her involvement. The man listens attentively, occasionally nodding or responding. The background remains consistent, with the floral wallpaper, framed pictures, and vintage decor contributing to the room's ambiance.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b33c10b98b42b37d36ecf6d415074e48a82f81d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f2bbf4922f79f36586c95f31d1be6fb407bc0e46a615ecf76188bcb3e3a5e6c +size 847802 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..0472c1e85126df9c43cc802c2c3623793c56d591 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The woman picks up a glass of water from the table and takes a sip, her actions adding a natural rhythm to the conversation. The man responds, his expression reflecting his engagement in the dialogue. The camera captures their interaction from a slightly different angle, emphasizing their facial expressions and the dynamics of their conversation. The room's details, including the white teapot on the table and the framed artwork on the walls, remain visible.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b65e732014124c3db6f09f475f81d170306071f6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:300e78ceed6fcb6fa6c98443eca2419a4d1011cc0bea4136e94ee4171fcb1c4d +size 1101484 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..660465d37b4d273afbff8bc9ab9de623e2f8f55a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the woman places the glass back on the table, her hands clasped together as she continues to speak. The man listens, his posture relaxed but attentive. The camera maintains a steady focus on their interaction, capturing the nuances of their conversation. The room's vintage charm, with its floral bedding, side table lamp, and shelves filled with decorative items, provides a consistent backdrop throughout the scene.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d753aeda6cc2bf1b12db935bc732e1ff15f14926 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaf0467665bfda4a307de4d077daf67e665a7168e0541b23c539bcd6970b9278 +size 1229280 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..589c89b8f77a911bdb169d8123be4d238e23c531 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/console.log new file mode 100644 index 0000000000000000000000000000000000000000..53cdc64aabd5218d239469ec0c1e7a8bbfbb3505 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/console.log @@ -0,0 +1,22 @@ +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/config.yaml +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.json +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:30:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:30:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:30:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:30:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..de4d79d431b36db451b6a78422af8a7081b8e526 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/debug.log @@ -0,0 +1,353 @@ +[02-20 12:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 12:16:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:16:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 12:16:10|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 12:16:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 12:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 12:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 12:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 12:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 12:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 12:16:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 12:16:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 12:16:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 12:16:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 12:16:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 12:16:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 12:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 12:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 12:16:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 12:16:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 12:16:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 12:16:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 12:16:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 12:16:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 12:16:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 12:16:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:16:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:16:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 12:16:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:16:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 12:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:16:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 12:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 12:16:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.05 s +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:16:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:16:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:16:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:16:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 12:16:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 12:16:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.47 s +[02-20 12:16:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:16:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:16:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/config.yaml +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 12:16:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the two men remain by the window, their expressions reflecting a sense of contemplation. Man A, with his dark blue jacket and mustache, keeps his hand near his collar, while Man B, in his brown suit, continues to look out thoughtfully. The camera subtly shifts to emphasize their profiles against the backdrop of the lattice windows and soft lighting, concluding the scene with a sense of quiet reflection.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.json +[02-20 12:16:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:16:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.mp4 +[02-20 12:16:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 12:16:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 12:16:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 12:16:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 12:30:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 12:30:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 12:30:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 12:30:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 12:30:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 12:30:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..07fe9eec3bab50028018400b016982fd55e54021 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with Man A, wearing a dark blue button-up jacket and sporting a mustache, seated at a desk in a dimly lit office. A green lamp and papers are on the desk, and a large map hangs on the wall behind him. Man B, dressed in a brown three-piece suit with a striped tie, stands beside him. The camera captures their initial positioning, focusing on the formal setting with wooden paneling and a filing cabinet in the background.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22bc74d44dce424b1b1f46382e98d89980e731e7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d35e58ae9c74334baadc5161c325d23a5938da92a4ea87a68a45f6b32a9f7688 +size 620891 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..029e27ee51bf90fcb358668a349074bddccbffb0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Man A rises from his chair and turns to face Man B, who remains standing. The camera follows their movement as they begin to walk toward the window. The office environment is detailed with a wooden chair, a green lamp, and soft lighting from table lamps. The transition sets the stage for their shift in focus from the desk to the window.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c779570dcd8979d2b12ffd9f46389c8dd3fd7819 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb876fa790d50f7934165eff9ad76076923919bf4fcd3631505fadb2025e9a6b +size 1089791 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..7579ccdcc958b938e0a4a053bfadc43e8d9d60ab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"As they approach the window, the camera pans to capture their profiles. Man A, in his dark blue jacket, adjusts his collar while looking out the window with a thoughtful expression. Man B, in his brown suit, stands beside him, also gazing outward. The intricate lattice design of the window and the brick column outside become prominent in the frame, adding depth to the scene.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..89d179b33d40c345fb991ae6253146f2a78d8b37 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7146626e4bf46d525238affbe31157c7e5bead2ab6ae95b63e46f5c409c4b4f9 +size 1049151 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..e73ba6a2423ed0637d4849db8966dad16cbb7bd7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The camera continues to follow Man A and Man B as they stand by the window. Man A maintains his posture, occasionally adjusting his collar, while Man B looks out pensively. The background includes additional table lamps and the lattice windows, reinforcing the formal and contemplative atmosphere of the office.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..864b53165eaac68cb8d192e4c455ed09713b5a28 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e52bfa48ad45a6e88dff889d4ad921951feca184dd4622d26702ca7bea9bde2b +size 1243754 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..b2ccbabd19d298b5d0659a91355aa991c32ef824 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the two men remain by the window, their expressions reflecting a sense of contemplation. Man A, with his dark blue jacket and mustache, keeps his hand near his collar, while Man B, in his brown suit, continues to look out thoughtfully. The camera subtly shifts to emphasize their profiles against the backdrop of the lattice windows and soft lighting, concluding the scene with a sense of quiet reflection.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..382531fc037b27e6f58c1993f0c86c52f32e0d31 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01afe0c77a71f25f5a90b3a7be3ef9fae3e487866f9f094e4abbdaef60b209eb +size 919555 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae17eb8c60829b0ef3c1dfa4ddb476bbed4c2866 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..2fb7a576767272dea42165d98f28e0752e062db5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/console.log @@ -0,0 +1,22 @@ +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:22:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:22:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/config.yaml +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.json +[02-20 11:22:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:36:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 11:36:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 11:36:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 11:36:12|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..43f89d839aa6ea4035734c8de2ebb6a8511686f9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/debug.log @@ -0,0 +1,353 @@ +[02-20 11:22:02|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 11:22:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:22:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 11:22:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 11:22:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 11:22:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 11:22:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 11:22:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 11:22:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 11:22:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 11:22:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 11:22:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 11:22:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 11:22:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 11:22:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 11:22:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 11:22:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 11:22:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 11:22:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 11:22:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 11:22:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 11:22:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 11:22:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 11:22:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 11:22:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 11:22:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 11:22:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 11:22:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 11:22:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 11:22:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 11:22:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 11:22:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:22:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 11:22:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:22:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 11:22:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 11:22:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 11:22:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 11:22:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 11:22:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 11:22:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 11:22:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 11:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 11:22:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 11:22:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.03 s +[02-20 11:22:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 11:22:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 11:22:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 11:22:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/config.yaml +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 11:22:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the man in the pink outfit continues to console the distressed individual, who remains in a hunched position. The woman in the qipao watches closely, her hand still resting on the man's shoulder, symbolizing her support. The scene concludes with the trio deeply engaged in their interaction, the man's gestures and the woman's attentive presence highlighting the emotional depth of the moment. The barred gate and brick walls in the background remain unchanged, maintaining the spatial consistency of the scene." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.json +[02-20 11:22:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.mp4 +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 11:22:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 11:22:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 11:22:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 11:36:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 11:36:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 11:36:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 11:36:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 11:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 11:36:12|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..b8c786315cf5fb4633fd045f14a55a7d2182c862 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a man in a light pink traditional Chinese outfit and a white hat seated on a wooden chair, facing a distressed individual in a brown robe. To his left, a woman in an ornate light pink qipao adorned with gold embroidery and jewelry sits quietly. The man starts to speak, gesturing with his hands as he interacts with the distressed person. The camera focuses on the trio, capturing the subtle movements and expressions that convey the gravity of the situation.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eeaedc7dcd88e014292dd4d1fa320f45fa2f078 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc886b15d30d3c6790c9706859f69a29f95e95e3dc1d17bc685b4de8bf2f6b11 +size 816441 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..feb606b33207f7a19b14ba01255d0c8316f1862b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the man in the pink outfit continues to engage with the distressed individual, now placing his hands on their shoulders in a gesture of reassurance. The woman in the qipao observes the interaction closely, her expression reflecting empathy. The man adjusts his posture slightly, maintaining eye contact with the distressed person, while the woman remains seated, her hand still resting on the man's shoulder. The camera subtly shifts to emphasize the emotional connection between the characters.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e613e80a14317a27b0054c23bcb9fe3f512eb177 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d916813197f23c62b5fa98e1bfb74efe2f19ebf7878a4a38cb27dd86340d5157 +size 1019272 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4b0e41f2f66f7fe093d82a56fa5c625a48c4b62d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The interaction deepens as the man in the pink outfit leans forward, offering further support to the distressed individual, who remains hunched over. The woman in the qipao watches attentively, her gaze fixed on the exchange. The man's gestures become more pronounced, indicating his earnest attempt to comfort the other person. The camera maintains a steady focus on the trio, ensuring that the viewer remains immersed in the emotional dynamics of the scene.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c9b3a234eee77467221a8536c9cf12b5702d7ab5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd67308ad0e006123731ec41653aad3c273fe875aecb4e00e821d976831c81b9 +size 1025451 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..49d1903f1a3b41f07430e4f8d3a1e3c454e0dd17 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the next segment, the man continues to provide comfort to the distressed individual, his hands gently placed on their back. The woman in the qipao remains seated, her expression conveying a mix of concern and compassion. The background, with its brick walls and barred gate, remains constant, reinforcing the confined and intimate nature of the setting. The camera angle subtly changes to capture the evolving interaction from different perspectives.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79da814abeb1bc9fa96a2fc7bc16e0ef0d64dbf4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29af1062efb374a03cbb875c2e1d42cc4939a7157a7dc512779bf4352f6ebd4d +size 1097074 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..d0b6eb5cad5f4030cab0f9eea4ce19fe05c2ce90 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the man in the pink outfit continues to console the distressed individual, who remains in a hunched position. The woman in the qipao watches closely, her hand still resting on the man's shoulder, symbolizing her support. The scene concludes with the trio deeply engaged in their interaction, the man's gestures and the woman's attentive presence highlighting the emotional depth of the moment. The barred gate and brick walls in the background remain unchanged, maintaining the spatial consistency of the scene.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4e059c93d25cf2e9071a2c2c9777f26592ddd90 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d5eb30c4e35cc5fcc6532862a1806ba0091c089e65f4888b11e2863916bae2d +size 1185820 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..783df39f1a3568425f48cd2283fc44c713136263 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/console.log new file mode 100644 index 0000000000000000000000000000000000000000..94d9bff461ecb4802d36d851d4d0378b5b5f6389 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/console.log @@ -0,0 +1,22 @@ +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/config.yaml +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.json +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:24:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:24:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:24:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:24:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4bf0b640dabb209e31d365bab6ed133f577cdb83 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/debug.log @@ -0,0 +1,353 @@ +[02-19 23:10:10|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 23:10:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:10:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 23:10:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 23:10:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 23:10:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 23:10:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 23:10:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 23:10:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 23:10:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 23:10:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 23:10:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 23:10:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 23:10:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 23:10:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 23:10:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 23:10:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 23:10:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 23:10:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 23:10:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 23:10:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 23:10:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 23:10:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 23:10:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 23:10:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 23:10:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 23:10:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 23:10:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 23:10:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 23:10:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 23:10:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 23:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 23:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:10:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 23:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 23:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 23:10:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 23:10:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:10:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.02 s +[02-19 23:10:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 23:10:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 23:10:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 23:10:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 23:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.65 s +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.66 s +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.66 s +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 23:10:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 23:10:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 23:10:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 23:10:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 23:10:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.80 s +[02-19 23:10:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 23:10:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 23:10:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/config.yaml +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 23:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the video concludes with the astronauts still engaged in their work on the spacecraft's exterior. The camera provides a final, sweeping view of their detailed suits and the complex machinery they interact with, with the Earth's vibrant colors and cloud patterns serving as a stunning backdrop. The astronauts' movements are captured in detail, ensuring that their identity and spatial consistency are maintained throughout the sequence, leaving a lasting impression of their dedication and skill in the vast expanse of space." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.json +[02-19 23:10:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 23:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.mp4 +[02-19 23:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 23:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 23:11:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 23:11:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 23:24:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 23:24:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 23:24:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 23:24:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 23:24:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 23:24:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..fd3960cb9b565e98083c3f1830e9f69f8ce84e9b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up shot of two astronauts in white space suits, featuring red and blue stripes on the arms, American flag patches on the right arm and left shoulder, and mission patches on the left chest area. They are floating in the vastness of space, surrounded by the blackness of the cosmos and scattered stars. The astronauts, wearing clear helmet visors and black gloves, begin by moving along the exterior of a spacecraft, their movements deliberate and focused. The camera follows them closely, capturing the intricate details of their suits and the complex machinery they interact with.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d4d8705405462c4607555f38cae7e9582f1b1885 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7bf525fa35e999972c051c9792cab197d75110a89210074e67365c012f2cace +size 1425814 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..56b13bd710d3a911831e5b8f95e2b1d1846cbd0d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the astronauts continue their work, the camera angle shifts slightly, providing a clearer view of the spacecraft's exterior panels and equipment. The astronauts, still in their white suits with distinctive markings, are seen using tools and manipulating components, their actions precise and methodical. The Earth begins to come into view in the background, its blue oceans and white clouds adding a vibrant contrast to the dark expanse of space. The camera maintains a steady follow, ensuring that the astronauts' detailed suits and the spacecraft's features are captured in detail.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b56744eab1a4b34c007d5410f1139c8b6063c1d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b5c91a9a7d07024d03ea720bde73e3a5824eef08abaa0116c4f19feca027345 +size 1792672 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..b896215b3c4af8a91dd99cb06e578f1a56f26d6b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The astronauts then move to a different section of the spacecraft, where they continue their meticulous work on the exterior panels. The camera provides a dynamic perspective, highlighting their interactions with the machinery and the intricate details of the spacecraft's structure. The Earth remains visible in the background, its vibrant colors and cloud patterns adding depth to the scene. The astronauts, still wearing their white suits with red and blue stripes, American flag patches, and mission patches, are seen working in tandem, their movements synchronized and purposeful.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6dcc94ed1f787816ba17d8ca1896c9ffc0b97739 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da0077d5d6e3f7234f655a9509d4e31532bcafe89d294637ad17569c221d7da5 +size 1310494 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..60227c670af0cf42c8f7672a9e91713238e94c98 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the sequence progresses, the camera captures the astronauts from various angles, ensuring that their detailed suits and the spacecraft's complex structure are consistently visible. The Earth's blue oceans and white clouds remain a constant backdrop, emphasizing the vastness of space and the astronauts' isolation. The astronauts continue their tasks with precision, their movements deliberate and focused, as they navigate the intricate machinery of the spacecraft.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..388f723e450ba5f6c141068a635ae7df4f61dc9d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0e2668da9f8689bdb55187632e391b57b181e27ae7d0e188945eb367216d508 +size 2236556 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..52e1327c930ce7f28feafe6e4b3ff5a317d51bd1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the video concludes with the astronauts still engaged in their work on the spacecraft's exterior. The camera provides a final, sweeping view of their detailed suits and the complex machinery they interact with, with the Earth's vibrant colors and cloud patterns serving as a stunning backdrop. The astronauts' movements are captured in detail, ensuring that their identity and spatial consistency are maintained throughout the sequence, leaving a lasting impression of their dedication and skill in the vast expanse of space.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..297ce6b46fcef1ee770a1597863e739c812ad790 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3959ff28ed9aec1b37363cec4588719b6eda1f359c72af4186f357c8b6cfd19d +size 1859488 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..916469ab8877d990cb8e02a3d697c95212f760d1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/console.log new file mode 100644 index 0000000000000000000000000000000000000000..dcda4e178167a057fdd188ae7d392a7f5223e4be --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/console.log @@ -0,0 +1,22 @@ +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:47:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:47:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/config.yaml +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.json +[02-19 20:48:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:01:38|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1c026865358b4bbe54da4ec9100d4d112c27acd7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/debug.log @@ -0,0 +1,353 @@ +[02-19 20:47:25|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 20:47:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 20:47:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 20:47:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 20:47:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 20:47:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 20:47:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 20:47:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 20:47:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 20:47:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 20:47:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 20:47:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 20:47:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 20:47:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 20:47:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 20:47:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 20:47:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 20:47:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 20:47:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 20:47:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 20:47:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 20:47:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 20:47:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 20:47:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 20:47:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 20:47:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 20:47:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 20:47:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 20:47:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 20:47:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 20:47:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 20:47:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 20:47:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 20:47:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:47:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:47:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 20:47:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:47:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:47:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:47:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 20:47:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:47:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 20:47:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:47:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 0.95 s +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 20:47:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 20:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 20:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.50 s +[02-19 20:48:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:48:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/config.yaml +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:48:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the man reaches the front of the room, continuing his walk past more individuals who are seated at desks. Another man in a dark shirt briefly enters the frame, leaning over a desk to assist one of the workers. The camera captures this interaction as the man in the light blue shirt continues his steady pace through the room.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.json +[02-19 20:48:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:48:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.mp4 +[02-19 20:48:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 20:48:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 20:48:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 20:48:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 21:01:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:01:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:01:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:01:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 21:01:38|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d0848b7f5a18eb9ac20714923e41df09998b1908 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a medium shot of the man standing near a desk filled with blueprints, a triangular ruler, and a green desk lamp. He is wearing a light blue short-sleeved shirt, black trousers, and glasses. He looks around the room, seemingly assessing the situation, before turning and beginning to walk away from the desk.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b99eecc838fe2c05c01cb3105d957929630bbf9b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5c63137765ea633e06526d4a76d8ab62413eca7274f76f6691a09564d1fbacc +size 473469 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..32d29dcffde531fb3a3c9244909c88c4c8db032d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the man walks away from the initial desk, the camera pans to follow his movement. He passes by other individuals seated at wooden desks, who are focused on their work. The room is decorated with large charts and Chinese calligraphy posters on the walls, and the natural light from the windows illuminates the space.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3735b72ccac452912a79142f4d11998338793c47 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09ce152b732a1c2e2f2c441739f1f41dc9ea9cc497c1406a19d73be50b0e939c +size 1906502 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..070e7843df1320fb6a0025070dc105e09b4084f9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The man continues walking deeper into the room, moving past more workers who are seated at desks, engrossed in their tasks. The camera maintains a steady follow shot, capturing his progression through the workspace. The background remains consistent, with wooden desks, large windows, and various charts and posters visible.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5cf1347bf4192013a74f4d943b78f7d1a81705b8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3912853099f83d58e24d0585306b4597dc912ece16db868513e48d8d699bac1 +size 1314344 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9f994b0c872531b0a627886099661325cc2036e3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The man approaches the front of the room, where additional workers are seated at desks, working on blueprints. The camera continues to follow him, keeping him centered in the frame. The natural light from the windows highlights the details of the workspace and the concentration of the workers.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5502abc22dee09d85a0337382bebcf51d2df30da --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:523046825ace9c2baf700e2fc6c4e7f1b57a5807e3a19f8050d9dfaff81d3fa0 +size 1087978 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..82c44f4578fc4e75215a1ca498b367412a58de77 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the man reaches the front of the room, continuing his walk past more individuals who are seated at desks. Another man in a dark shirt briefly enters the frame, leaning over a desk to assist one of the workers. The camera captures this interaction as the man in the light blue shirt continues his steady pace through the room.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d98365546de4a84a96170e6e0bda7b1944cc042 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70dc94102cd37225f3cbc8d89096be673f8a2231b9a0be0e9ab2b1675e2cdd62 +size 1287097 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f463362f409d5a442c36c9be1985947928f00de0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/console.log new file mode 100644 index 0000000000000000000000000000000000000000..09c5640c498696f4ebf17320efcb5041b903fbde --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/console.log @@ -0,0 +1,22 @@ +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:10:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:10:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/config.yaml +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.json +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:24:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:24:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:24:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:24:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..65c60071a6d611536936a4a0633323559d2793d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/debug.log @@ -0,0 +1,353 @@ +[02-20 16:10:42|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 16:10:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:10:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 16:10:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 16:10:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 16:10:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 16:10:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 16:10:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 16:10:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 16:10:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 16:10:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 16:10:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 16:10:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 16:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 16:10:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 16:10:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 16:10:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 16:10:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 16:10:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 16:10:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 16:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 16:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 16:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 16:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 16:10:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 16:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 16:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 16:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 16:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 16:10:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 16:10:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 16:10:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:10:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 16:10:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:10:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 16:10:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 16:10:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 16:10:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 16:10:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 16:10:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 16:10:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 16:10:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 16:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 16:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 16:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.40 s +[02-20 16:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 16:11:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 16:11:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/config.yaml +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 16:11:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the man on the right concludes his explanation, holding the tool still as he looks directly at the man on the left. The man on the left nods slightly, signaling comprehension or agreement. The scene ends with both men standing in the same positions, the industrial environment unchanged, emphasizing the continuity and focus of their interaction throughout the video.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.json +[02-20 16:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 16:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.mp4 +[02-20 16:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 16:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 16:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 16:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 16:24:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 16:24:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 16:24:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 16:24:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 16:24:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 16:24:53|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0f5a8bfb82cea2f4c94e1f69ab31b4a764974bdb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with the two men positioned in an industrial facility, surrounded by large stainless steel tanks and red piping. The man on the left, sporting dark curly hair and light stubble, is dressed in a dark olive-green jacket. He stands calmly, facing the man on the right, who is bald, wears glasses, and has a goatee, dressed in a dark plaid shirt. The man on the right holds a long-handled tool with a mesh scoop, beginning to gesture with it as he speaks, drawing the attention of the other man.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..911d64cbb1597559167fadadb6e73d3439299779 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18fde433dc942dc90d8f9c44cf9b72f4b7e7e97c174abd37fd0486dbcdd660f1 +size 1038226 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5ef167ab9266e843462c38b1b914191c6874178c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the man on the right becomes more expressive with his tool, moving it in various directions to illustrate his points. His facial expressions become more animated, suggesting he is explaining something important. The man on the left remains attentive, his posture steady, occasionally nodding in response. The background, with its industrial equipment and control panels, remains unchanged, maintaining the setting's authenticity.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e2ee7f85b6d3805ada2d3d382aad924e84a1b7a0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27de3b89c0fad15492a38ad9e03c23fd5467e9c75d0961e67447305d8bb9bc35 +size 1054578 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..5bfdd34c295ee4855d717a5e03e049c19dffd052 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The interaction continues with the man on the right using the tool to point towards the large tank behind them, possibly highlighting a specific aspect of their work. His gestures are deliberate, and his tone seems to convey urgency or importance. The man on the left listens intently, his expression reflecting engagement and understanding. The camera stays focused on the two, ensuring their interaction remains the central element of the scene.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f9bd451400f72b4434532df5d23a283771ae4bcd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d608a551f3547503e62ba7225feec66cfd9e73c94e153ffbd17f44274ef2cca +size 736508 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..6f09421ec7d9eb3b9d5d4153f324852ad777d167 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Moving into the next phase, the man on the right shifts the tool's position, now holding it closer to his body while continuing his explanation. His gestures are less expansive but remain purposeful. The man on the left maintains his stance, his gaze fixed on the other man, indicating active participation in the conversation. The industrial setting, with its metallic surfaces and red pipes, provides a consistent and immersive backdrop.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c1a7114fb01e2561cb15db0f8e1875ee7bd2bc5c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6562072abe8cde4d25dab79fbeb661e1a172f381dd981df28824470e8003cb45 +size 942323 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..12b9bdf67e364864aa0ed5c2458611e7624fcd43 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the man on the right concludes his explanation, holding the tool still as he looks directly at the man on the left. The man on the left nods slightly, signaling comprehension or agreement. The scene ends with both men standing in the same positions, the industrial environment unchanged, emphasizing the continuity and focus of their interaction throughout the video.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf6e4fb122fc5fc6d65253b80d13d55e1b43ebce --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f8bc42695f618ec616b15072738627d1ce1ad01abb371690e34fc464ee30744 +size 749811 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75071fd073ca325e8343c3a470f455b8aefed933 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/console.log new file mode 100644 index 0000000000000000000000000000000000000000..a3e75dd52f2fd0cd7f9f546f28453cc6893bd9d8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/console.log @@ -0,0 +1,22 @@ +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-21 00:19:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-21 00:19:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/config.yaml +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.json +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-21 00:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-21 00:33:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-21 00:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-21 00:33:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..37c2af3ff89f22141d4f6159bacf47a4ad9c8b81 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/debug.log @@ -0,0 +1,353 @@ +[02-21 00:19:06|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-21 00:19:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-21 00:19:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-21 00:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-21 00:19:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-21 00:19:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-21 00:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-21 00:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-21 00:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-21 00:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-21 00:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-21 00:19:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-21 00:19:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-21 00:19:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-21 00:19:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-21 00:19:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-21 00:19:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-21 00:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-21 00:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-21 00:19:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-21 00:19:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-21 00:19:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-21 00:19:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-21 00:19:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-21 00:19:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-21 00:19:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-21 00:19:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-21 00:19:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-21 00:19:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-21 00:19:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-21 00:19:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-21 00:19:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-21 00:19:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-21 00:19:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-21 00:19:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-21 00:19:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-21 00:19:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-21 00:19:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-21 00:19:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-21 00:19:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-21 00:19:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-21 00:19:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-21 00:19:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-21 00:19:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-21 00:19:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.52 s +[02-21 00:19:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-21 00:19:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-21 00:19:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/config.yaml +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-21 00:19:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final moments, the man's expression remains engaged, with his mouth slightly open as if he is continuing to speak. His gaze is directed slightly to the right, and the background audience is still out of focus. The lighting continues to highlight the texture of his suit and the details of his face, emphasizing the stress and fatigue evident in his appearance." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.json +[02-21 00:19:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-21 00:19:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.mp4 +[02-21 00:19:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-21 00:19:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-21 00:19:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-21 00:19:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-21 00:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-21 00:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-21 00:33:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-21 00:33:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-21 00:33:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-21 00:33:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2204431aef7a6df5d0d6236e4b1f58b6b410c82b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of a man in a gray pinstripe suit jacket over a white collared shirt, his short dark hair neatly styled. His face shows signs of stress, with visible sweat on his forehead and slight dark circles under his eyes. He is seated in a formal setting, likely a courtroom or auditorium, with a blurred audience in the background. The lighting is soft, highlighting the texture of his suit and the details of his face.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..348e431b15a8ceb08ffad6167c0f37d119ad2ab5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ae3a53970f1ee44d557dde8015e86aeabb4b2b3918da58a1e389f694852f722 +size 417606 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2f5b4cd1e1904c8311ccd1825e1e5eae134e01b7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the man subtly shifts his gaze, looking slightly to the left and then back to the center, indicating he is focused on something or someone off-camera. His expression remains serious and contemplative, with minimal movement, suggesting he is deeply engaged in the situation.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41de575bc0ebd919f48915ac06c6b665d33d7888 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0df60f12f421c2595b3706c7bc965e3b48b2de803b7004a386b06f5f284f06d +size 500096 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a921ab800f7e0c6115ed1c3b4b3d7de12187bf59 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The man's gaze continues to shift slightly, now looking more directly forward. His lips move subtly, suggesting he might be about to speak or reacting to something being said. The background audience remains out of focus, maintaining the sense of depth and context. The lighting continues to emphasize the texture of his suit and the details of his face.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15049c31cd4add5240f67d3ea5cf295064f2f2f4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2edd34579fe85aa1ebd595f163252e71e5dd03035ce4964c0d85e1453e19649 +size 524346 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ebd99dfb9a19f9674759d4d85171e3cfed0faadf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The man's expression becomes more animated, with his mouth opening slightly as if he is speaking or responding to a question. His gaze shifts slightly to the right, indicating a change in focus or reaction. The camera remains steady, keeping his face in sharp focus, capturing the subtle changes in his expression and the slight sheen of sweat on his forehead.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..19d6df0b02fbd802ae422009d39d124e7637dd7b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f991cb18ba47b1064856d5358a88cb944b2c005cd53b74aa4531c9076b42044a +size 594454 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ba9def88ac9c95e00a08a25dde7dce90460bd463 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments, the man's expression remains engaged, with his mouth slightly open as if he is continuing to speak. His gaze is directed slightly to the right, and the background audience is still out of focus. The lighting continues to highlight the texture of his suit and the details of his face, emphasizing the stress and fatigue evident in his appearance.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..91ccde2feb2f09e164c3580dbb682a5d1ed20135 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c6799f90e9e7c4784bd87bdb100a214e4bcf9adcb5bde62139366f5b61e87b0 +size 495192 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9664e4be2e09054700c13b5722a2cd456963b7ab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/console.log new file mode 100644 index 0000000000000000000000000000000000000000..2334a26a587d54b7f1f32243a9e934ffba833d52 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/console.log @@ -0,0 +1,22 @@ +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:36:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:36:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/config.yaml +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.json +[02-20 00:36:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:49:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:49:58|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a2fd6facac6b0657342845ad68cccbde8d2de986 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/debug.log @@ -0,0 +1,353 @@ +[02-20 00:35:51|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 00:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:35:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:36:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 00:36:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 00:36:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 00:36:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 00:36:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 00:36:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 00:36:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 00:36:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 00:36:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 00:36:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 00:36:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 00:36:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 00:36:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 00:36:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 00:36:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 00:36:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 00:36:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 00:36:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 00:36:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 00:36:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 00:36:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 00:36:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:36:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:36:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 00:36:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:36:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:36:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:36:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 00:36:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:36:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 00:36:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:36:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:36:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 00:36:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.26 s +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.66 s +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.67 s +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.67 s +[02-20 00:36:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:36:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 00:36:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 00:36:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.93 s +[02-20 00:36:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:36:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:36:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 00:36:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/config.yaml +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 00:36:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the man and woman conclude their interaction. The man’s smile softens into a contented expression, and he looks directly at the camera for a brief moment before returning his gaze to the object of their attention. The woman, still smiling broadly, turns her head fully toward the camera, her eyes meeting the viewer’s. Her demeanor suggests she is happy and satisfied with the moment. The camera remains steady, capturing their expressions against the backdrop of blurred green foliage.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.json +[02-20 00:36:39|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:36:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.mp4 +[02-20 00:36:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 00:36:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 00:36:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 00:36:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 00:49:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 00:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:49:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:49:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:49:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 00:49:58|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3e84bf4121a859d3a3e0b55c956d349b03c1703d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a close-up of a man and a woman standing outdoors. The man, on the left, has short dark hair, wears round patterned glasses (black and white frames), a black turtleneck shirt, and a light grey blazer. The woman, on the right, has long dark hair with a small braid at the front and is wearing a white collared shirt under a light grey blazer. Both are looking off to the right, seemingly focused on something out of frame. The background is blurred green foliage, suggesting a garden or park setting. The camera is steady, capturing their upper bodies and facial expressions.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a06af9e04c0072341baa6c43e69dd119f23fb11 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fb8a13b65bf64c36df0cff2354cf0f05a5a4967393d527c0c408c55fa921c61 +size 686358 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..d481f61a9cb79f6bca19aa4bc692c5b3db93823f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the man and woman maintain their positions and gazes. The man’s expression changes subtly as he appears to begin speaking or reacting verbally, his mouth moving while his eyes remain fixed on the object of their attention. The woman starts to smile faintly, her head tilting slightly as if she finds the situation amusing or intriguing. The background remains unchanged, with the blurred greenery providing a consistent natural backdrop.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e787f5b791daf528b79c02fa7af134edb56b330 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a3c230c8db599eb6b441adb21af0c0f8fbe41e54cac4953858859b98ec3d468 +size 829326 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..f9ab9e41a6efab00940f8dc07fbc5b59f688cf46 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The interaction continues with the man and woman still looking off to the right. The man’s expression softens further, transitioning into a slight smile as he seems to share in the woman’s amusement. The woman’s smile widens, and she turns her head slightly toward the camera while still glancing back at the object of their attention. Her body language suggests she is enjoying the moment. The camera remains steady, keeping both individuals in focus against the backdrop of green foliage.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fbc86cd09d8cfb208f12a66484c53fa2a7229cb3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e0512146d492259a76a0f529ed81e907fb1e3f1b7e9ba424f3850c0a4e048d9 +size 826750 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..122fb8c67062755a3ce53169db0e6193f1181b90 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the next segment, the man and woman continue their interaction. The man’s smile becomes more pronounced, indicating a shared sense of humor or enjoyment. The woman, now fully smiling, turns her head slightly more toward the camera, her eyes sparkling with amusement. She glances back at the object of their attention one last time before fully engaging with the viewer. The background remains consistent, with the blurred greenery providing a natural and serene setting.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da281b3ebd306a4a759816857a8a9e572815d276 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e91302678974e4602d2bbfbd1f4c73cd2746559eebde826d0eb79c137fe5f8b +size 1162712 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..77dd091c9147c33e75e00379b1ea5d73a530cf8a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the man and woman conclude their interaction. The man’s smile softens into a contented expression, and he looks directly at the camera for a brief moment before returning his gaze to the object of their attention. The woman, still smiling broadly, turns her head fully toward the camera, her eyes meeting the viewer’s. Her demeanor suggests she is happy and satisfied with the moment. The camera remains steady, capturing their expressions against the backdrop of blurred green foliage.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b3a1e32799e356c77ef599b1cd7bb543daa5584 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d40093b44793777df4666e57378f32c999e265612c692cd100d922e2e433d313 +size 969092 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31be2c786f0f81c80e7a18dc18e09d97a5b39d51 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/console.log new file mode 100644 index 0000000000000000000000000000000000000000..a4c41010d8ea0ca9cd4c0ab7d35fe140c48af435 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/console.log @@ -0,0 +1,22 @@ +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/config.yaml +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.json +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:29:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:29:27|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4daf0eee8c95e4880b9b9df2b826a3d2e21dc39f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/debug.log @@ -0,0 +1,352 @@ +[02-20 04:15:13|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 04:15:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:15:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 04:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 04:15:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 04:15:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 04:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 04:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 04:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 04:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 04:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 04:15:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 04:15:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 04:15:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 04:15:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 04:15:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 04:15:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 04:15:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 04:15:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 04:15:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 04:15:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 04:15:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 04:15:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 04:15:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 04:15:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 04:15:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 04:15:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 04:15:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 04:15:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 04:15:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 04:15:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 04:15:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 04:15:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 04:15:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 04:15:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 04:15:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 04:15:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 04:15:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 04:15:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 04:15:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 04:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 04:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:15:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 04:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 04:15:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 04:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 04:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-20 04:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 04:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 04:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 04:15:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 04:15:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 04:15:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 04:15:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 04:15:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 04:15:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.19 s +[02-20 04:15:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 04:16:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 04:16:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/config.yaml +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 04:16:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video starts with a close-up side profile of an older man wearing a light-colored cowboy hat and an olive-green jacket. He is standing outdoors, gazing toward the right with a thoughtful expression. The background shows distant mountains under a soft, hazy sky. The man then slowly turns his head to the right, revealing more of his face, including wrinkles on his neck and cheek. Continuing from this, he turns back to show the back of his head and hat, maintaining the same serene outdoor setting throughout.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.json +[02-20 04:16:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 04:16:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 04:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 04:16:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 04:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 04:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 04:29:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 04:29:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 04:29:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 04:29:27|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..36348c3bfad4c9be6311eb0dfe8d98935a588bdd --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a close-up side profile of an older man wearing a light-colored cowboy hat and an olive-green jacket. He is standing outdoors, gazing toward the right with a thoughtful expression. The background shows distant mountains under a soft, hazy sky. The man then slowly turns his head to the right, revealing more of his face, including wrinkles on his neck and cheek. Continuing from this, he turns back to show the back of his head and hat, maintaining the same serene outdoor setting throughout.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e407c947fada82e90f2efc5c5999b14cd5f04049 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd8e5591a7ee5c97608ee757b87d60c167201ec3dc84c40d55b6f3c731909111 +size 430958 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..220bad087f13eabb56875e2c834f46ae6923dff8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/console.log new file mode 100644 index 0000000000000000000000000000000000000000..32818d69b29f3ceaff02d23f4583a3d75f2b4023 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/console.log @@ -0,0 +1,22 @@ +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/config.yaml +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.json +[02-20 02:16:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:29:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:29:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:29:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:29:56|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f2342153d9b78c9831883df6a460823992d4edb1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/debug.log @@ -0,0 +1,352 @@ +[02-20 02:15:49|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 02:15:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:15:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 02:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 02:15:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 02:15:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 02:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 02:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 02:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 02:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 02:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 02:15:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 02:15:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 02:15:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 02:16:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 02:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 02:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 02:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 02:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 02:16:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 02:16:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 02:16:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 02:16:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 02:16:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 02:16:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 02:16:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 02:16:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 02:16:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 02:16:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:16:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 02:16:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 02:16:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 02:16:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.29 s +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.79 s +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.79 s +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.79 s +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 02:16:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 02:16:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 02:16:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 02:16:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.76 s +[02-20 02:16:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 02:16:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 02:16:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 02:16:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/config.yaml +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 02:16:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The scene is set in a rustic room with cracked mud walls and a woven basket hanging on the wall. A man lies in bed, wearing a light-colored shirt with subtle patterns, his expression suggesting he is either speaking or reacting to something. To his right stands a woman dressed in a light green floral-patterned traditional Chinese dress, her hair neatly pulled back, watching the interaction with a concerned expression. On the other side of the bed, a man in an olive green jacket over a white shirt holds a red folded item, possibly a cloth or small bag. As the video progresses, the man in the jacket begins to carefully unfold the red item, revealing its contents, while the man in bed turns his head to look at it, and the woman remains focused on the unfolding event.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.json +[02-20 02:16:38|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 02:16:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 02:16:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 02:16:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 02:29:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 02:29:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 02:29:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 02:29:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 02:29:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 02:29:56|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2e4afad352bd0de823c0c8a0bedd072ee2d137b5 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The scene is set in a rustic room with cracked mud walls and a woven basket hanging on the wall. A man lies in bed, wearing a light-colored shirt with subtle patterns, his expression suggesting he is either speaking or reacting to something. To his right stands a woman dressed in a light green floral-patterned traditional Chinese dress, her hair neatly pulled back, watching the interaction with a concerned expression. On the other side of the bed, a man in an olive green jacket over a white shirt holds a red folded item, possibly a cloth or small bag. As the video progresses, the man in the jacket begins to carefully unfold the red item, revealing its contents, while the man in bed turns his head to look at it, and the woman remains focused on the unfolding event.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40a284faebafe274418cc8326bc994ca5bc87167 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0be53d846d5d75a94eeaddbd13b5e777dc94d27eb03bd010e1421f71623cdced +size 1206034 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..225de1cfeb24673f6e3f181a5112616ef44505a8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b2a3b008f8922f6ec1070f9e225467f1f5f9dc79 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/console.log @@ -0,0 +1,22 @@ +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 09:53:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 09:53:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 09:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/config.yaml +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.json +[02-20 09:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 10:07:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:07:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:07:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:07:16|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7ab0ca54321848e27913ebd295ffef059fdc507a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/debug.log @@ -0,0 +1,353 @@ +[02-20 09:53:10|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 09:53:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 09:53:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 09:53:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 09:53:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 09:53:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 09:53:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 09:53:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 09:53:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 09:53:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 09:53:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 09:53:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 09:53:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 09:53:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 09:53:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 09:53:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 09:53:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 09:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 09:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 09:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 09:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 09:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 09:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 09:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 09:53:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 09:53:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 09:53:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 09:53:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 09:53:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 09:53:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 09:53:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 09:53:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 09:53:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 09:53:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 09:53:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 09:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 09:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 09:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 09:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 09:53:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 09:53:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 09:53:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 09:53:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 09:53:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 09:53:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 09:53:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 09:53:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 09:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 09:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 09:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 0.93 s +[02-20 09:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 09:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 09:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 09:53:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 09:53:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 09:53:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 09:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 09:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 09:53:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 09:53:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 09:53:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 09:53:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.96 s +[02-20 09:53:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 09:53:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 09:53:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 09:53:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/config.yaml +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 09:53:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final moments of the video, the elderly woman is still speaking, her head turned to the right. Her facial expressions continue to convey her engagement in the conversation. The younger woman maintains her attentive posture, her eyes occasionally meeting the elderly woman's, reinforcing the sense of communication. The background, with its plain wall and red-framed alcove, remains consistent, ensuring the focus stays on the two women and their interaction." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.json +[02-20 09:53:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.mp4 +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 09:53:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 09:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 09:53:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 10:07:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 10:07:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 10:07:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 10:07:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 10:07:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 10:07:16|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..dc248a457717a8c1570e40d85a185f79746eb944 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with the two women standing side by side. The elderly woman, with her gray hair in a bun and wearing a dark top with an ornate collar, is positioned on the left. The younger woman, with short brown hair and a light pink collared shirt, stands on the right. The background features a plain wall with a red-framed alcove, adding a subtle detail to the setting. The elderly woman starts to turn her head slightly to her right, preparing to speak.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e738bc0ac1176acdd62e359b372f4339544a3bf3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7683e2bc46cd37d4f693f2716846246719d8b4b1fb4da1435d6cef43669b0dd0 +size 1211600 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..61b9f7f8ac2b82e24bea7e2fe86c0a15094c3c9b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the elderly woman continues to turn her head further to the right, her facial expressions begin to change, indicating she is starting to speak. The younger woman remains mostly still, her gaze directed forward but with a slight shift towards the elderly woman, suggesting she is listening. The red-framed alcove in the background remains a constant element, providing a stable backdrop for the scene.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c2f59f3af39aff3fef54a4d68e251c2334d675bb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ecbd384ed5ff5186abef89940d047a03565ad5b0781490aa7821281cc5d49a5 +size 1227367 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..11dce4d5a27d4a9f681c095685980debae5ccf5e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The elderly woman is now fully engaged in speaking, her head turned more significantly to the right. Her facial expressions are more animated, reflecting the content of her words. The younger woman maintains her attentive posture, her eyes occasionally meeting the elderly woman's, reinforcing the sense of communication. The background, with its plain wall and red-framed alcove, remains unchanged, keeping the focus on the interaction between the two women.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b816a573f07fff456120ccfb3ece2bac24c3471c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8569a8bddb3b8736d1a746f51b549af69db9fe476c02b6095ba0481d63418910 +size 1152599 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..035e92c06092b161b6e56d28e485dd1b2512b2d8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The elderly woman continues her conversation, her head still turned to the right. Her expressions remain engaged, showing her active participation in the dialogue. The younger woman stays in her attentive stance, her gaze occasionally shifting towards the elderly woman, indicating ongoing listening. The red-framed alcove in the background remains a consistent element, providing a stable visual context.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f3587688509f76138f879f9088a3550609bb04c9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af1c5c8787261bfa10b31154799e0d4a792cb252539ff66efaf746750845dc46 +size 1029351 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..779a819d73e181d0ada6c9495016ba290823dfef --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments of the video, the elderly woman is still speaking, her head turned to the right. Her facial expressions continue to convey her engagement in the conversation. The younger woman maintains her attentive posture, her eyes occasionally meeting the elderly woman's, reinforcing the sense of communication. The background, with its plain wall and red-framed alcove, remains consistent, ensuring the focus stays on the two women and their interaction.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..364468ecb3866d2bda3099725ab3eb788b5688e1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5612f66497f493517003ddcca0527d204fd24e8603852a2bb35032483a3c44f4 +size 981489 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..298f9b7c3dcbee29f60e66fdb1faaf30930e749c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c22629a20b176030a1eb48a6888650724f0ec720 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/console.log @@ -0,0 +1,22 @@ +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/config.yaml +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.json +[02-20 13:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:41:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:41:19|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4a73f067600dfbfa8a136ea23c346ca7067b9c12 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/debug.log @@ -0,0 +1,352 @@ +[02-20 13:27:13|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 13:27:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:27:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 13:27:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 13:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 13:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 13:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 13:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 13:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 13:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 13:27:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 13:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 13:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 13:27:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 13:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 13:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 13:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 13:27:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 13:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 13:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 13:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 13:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 13:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 13:27:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 13:27:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 13:27:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 13:27:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 13:27:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 13:27:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:27:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 13:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:27:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 13:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:27:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 13:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 13:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 13:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 13:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 13:27:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:27:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 13:27:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 13:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 13:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:27:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:27:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 13:27:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 13:27:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.07 s +[02-20 13:27:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:27:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:27:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 13:27:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/config.yaml +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 13:28:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a wide, static shot of two individuals in flowing white traditional attire standing on a small wooden raft in a calm river. The background is dominated by lush green trees and towering rocky cliffs, which are mirrored perfectly in the still water below. The individuals' garments are adorned with intricate patterns, and they wear elaborate headpieces or hair accessories. The camera remains fixed, capturing the peaceful ambiance of the scene. As the video continues, the perspective shifts to a close-up of one individual's foot, showcasing the light-colored, soft-soled footwear stepping delicately onto a surface covered with scattered petals or leaves, ensuring the continuity of their attire and the natural setting throughout the sequence." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.json +[02-20 13:28:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 13:28:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 13:28:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 13:28:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 13:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 13:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:41:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:41:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:41:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 13:41:19|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a919fcb13eb8045ac140010cec49a58ad39e10d2 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a wide, static shot of two individuals in flowing white traditional attire standing on a small wooden raft in a calm river. The background is dominated by lush green trees and towering rocky cliffs, which are mirrored perfectly in the still water below. The individuals' garments are adorned with intricate patterns, and they wear elaborate headpieces or hair accessories. The camera remains fixed, capturing the peaceful ambiance of the scene. As the video continues, the perspective shifts to a close-up of one individual's foot, showcasing the light-colored, soft-soled footwear stepping delicately onto a surface covered with scattered petals or leaves, ensuring the continuity of their attire and the natural setting throughout the sequence.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61db9f44d79e8d01ec478698f1deb534b46af4f6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94630a4e53b8b92215b84daf62edaee8a958e756075119f65cfd7a466d7dc056 +size 939628 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd2ef1d95b4a9063462a696a47a1561bfdfb2987 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/console.log new file mode 100644 index 0000000000000000000000000000000000000000..13df0e8ceb5f83352a00b9105609219fe1b1a037 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/console.log @@ -0,0 +1,22 @@ +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:37:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:37:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/config.yaml +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.json +[02-20 06:37:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:51:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:51:07|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4d2fb22b7f63218450ff6508b83072763776e3fe --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/debug.log @@ -0,0 +1,352 @@ +[02-20 06:36:58|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 06:37:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:37:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 06:37:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 06:37:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 06:37:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 06:37:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 06:37:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 06:37:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 06:37:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 06:37:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 06:37:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 06:37:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 06:37:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 06:37:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 06:37:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 06:37:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 06:37:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 06:37:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 06:37:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 06:37:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 06:37:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 06:37:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 06:37:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 06:37:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 06:37:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 06:37:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 06:37:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 06:37:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 06:37:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 06:37:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:37:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 06:37:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:37:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 06:37:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 06:37:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 06:37:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 06:37:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:37:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-20 06:37:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 06:37:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 06:37:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 06:37:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 06:37:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.65 s +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.65 s +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 06:37:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 06:37:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 06:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 06:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 06:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.85 s +[02-20 06:37:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 06:37:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 06:37:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 06:37:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/config.yaml +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 06:37:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video begins with a close-up of a woman with short, straight black hair and bangs, wearing a black top or jacket. Her silver-colored dangling earrings with circular pendants and light purple nail polish on her visible fingers are clearly visible. She holds a white flip phone with a blue screen to her ear, her expression serious and focused as if listening intently. The background shows a softly lit, blurred interior space with a patterned, beige armchair and warm ambient lighting. As the video progresses, she continues her conversation, her demeanor unchanged. In the final segment, she lowers the phone from her ear, her gaze shifting downward, indicating a moment of contemplation or concern.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.json +[02-20 06:37:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 06:37:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 06:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 06:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 06:51:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 06:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 06:51:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 06:51:07|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f6d03569496e70b3238e7999caead96c76c38376 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a close-up of a woman with short, straight black hair and bangs, wearing a black top or jacket. Her silver-colored dangling earrings with circular pendants and light purple nail polish on her visible fingers are clearly visible. She holds a white flip phone with a blue screen to her ear, her expression serious and focused as if listening intently. The background shows a softly lit, blurred interior space with a patterned, beige armchair and warm ambient lighting. As the video progresses, she continues her conversation, her demeanor unchanged. In the final segment, she lowers the phone from her ear, her gaze shifting downward, indicating a moment of contemplation or concern.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..28c877eb594cb7801a8969bb79e78a3b0c576cad --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ded2599624ca25b3d64514e41e05f3474bcc285de40b15c4d7f6bf65e54183a5 +size 362888 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..706d4d9b033c32a8d3c9191d4286427bd493a551 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/console.log new file mode 100644 index 0000000000000000000000000000000000000000..eb471218adc15879786aa0d31c8ae74a322a6ba0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/console.log @@ -0,0 +1,22 @@ +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/config.yaml +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.json +[02-20 08:41:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:54:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:54:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:54:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:54:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bcf5a29bb0f54442d3895a2b12f6d21d91faf7c8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/debug.log @@ -0,0 +1,353 @@ +[02-20 08:40:44|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 08:40:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:40:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 08:40:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 08:40:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 08:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 08:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 08:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 08:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 08:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 08:40:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 08:40:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 08:40:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 08:40:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 08:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 08:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 08:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 08:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 08:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 08:40:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 08:40:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 08:40:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 08:40:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 08:40:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:40:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:40:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 08:40:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:40:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 08:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:40:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 08:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:40:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 08:40:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:41:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:41:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:41:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:41:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 08:41:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 08:41:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.25 s +[02-20 08:41:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:41:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:41:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/config.yaml +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 08:41:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final moments, the woman finishes brushing the child's hair, her movements gentle and deliberate. The reflection in the mirror shows her turning slightly, offering a different angle of her face and the child's braids. The man in the background remains seated, his reflection still visible. The soft, natural light continues to illuminate the room, creating a peaceful and tender conclusion to the scene." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.json +[02-20 08:41:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:41:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.mp4 +[02-20 08:41:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 08:41:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 08:41:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 08:41:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 08:54:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 08:54:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 08:54:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 08:54:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 08:54:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 08:54:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ab824d9354ecaa7bcb9f9e56db39c95b35d3650a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a woman in a green and white striped dress standing in front of a large, ornate mirror, brushing the braided hair of a young girl seated before her. The woman's dark skin and neatly styled bun are prominent, and her dress fits snugly at the waist. The reflection in the mirror shows her focused expression as she gently brushes the child's hair. The room is dimly lit with soft, natural light coming from a window to the right, creating a serene atmosphere. In the background, a man in period clothing is partially visible in the mirror's reflection, seated on a chair.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2cb0a251133d9295ee32a128a86de59c25e5b172 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc17115822a409ce4d882208a77c0e07ee877b11bc572c8e0b437e6cb99b561b +size 1028405 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b1c64b4e3d5b9a5dce9ea4ae466c689b91b8c9a3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the woman continues to brush the child's hair, her movements deliberate and gentle. The camera subtly shifts, revealing more of the room's vintage decor, including a table with various items and the man in the background, whose reflection is now more visible. The woman's expression remains focused, and the intricate details of her dress and the texture of the child's braids are highlighted by the soft lighting.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d966e4c2ecf4663bcc435754582f6372418f2af7 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fc95acb1f141879382fcf76fe20d8abdf25265cd04e8a0e19b7c2a72cc98722 +size 1020795 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..40bce50209f15159d9f8bb298fe86d4b74927e48 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The woman continues her task, occasionally turning slightly to get a better angle for brushing the child's hair. The reflection in the mirror shows her adjusting her position, maintaining her focus on the child. The man in the background remains seated, his presence adding depth to the scene. The soft, natural light continues to illuminate the room, emphasizing the intimate and caring nature of the moment.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..651893cf8ce3362eb9118cf322f73f1f5da2817b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b637b4d48b37ed71b306308a6411fae463cb3bc719e91c2fb600f3d08690749 +size 998996 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..623c2e65d796062fe86d2fcb835e2e2f395532d1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The woman completes brushing the child's hair, her attention fully on the task. The reflection in the mirror captures her final strokes and the serene expression on her face. The man in the background remains seated, his reflection still visible. The overall atmosphere remains calm and intimate, with the vintage room setting and soft lighting enhancing the sense of timelessness and care.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f8c9c47ba39e647f117c122eca39122c1c980b4c --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8ea22fb5616a01fbac8ac5851be7b0b999ba45e4703010defc16109e153a43b +size 1083021 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..50094ca42910c53e911f235549df1794d0aedc8b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments, the woman finishes brushing the child's hair, her movements gentle and deliberate. The reflection in the mirror shows her turning slightly, offering a different angle of her face and the child's braids. The man in the background remains seated, his reflection still visible. The soft, natural light continues to illuminate the room, creating a peaceful and tender conclusion to the scene.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ce9de50c511ed4dd3ed8e04eb8bdffc1786c3a93 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b8e9c77f03d45cc8f522cf6820b4cc92684825c6d9993b6a6e84d8117e3cd4 +size 1201676 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1878d7b3a6a301300d798772ab8a3ae99f690e26 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/console.log new file mode 100644 index 0000000000000000000000000000000000000000..6b2de1c55da16f7edcf45c6dffceed1ea64bd883 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/console.log @@ -0,0 +1,22 @@ +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 18:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 18:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 18:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 18:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/config.yaml +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.json +[02-20 18:39:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 18:53:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 18:53:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 18:53:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 18:53:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4f88d65c85cc8b0ee51e7b99b9794a0514dffe65 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/debug.log @@ -0,0 +1,353 @@ +[02-20 18:38:37|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 18:38:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 18:38:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 18:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 18:38:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 18:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 18:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 18:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 18:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 18:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 18:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 18:38:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 18:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 18:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 18:38:46|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 18:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 18:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 18:38:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 18:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 18:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 18:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 18:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 18:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 18:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 18:38:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 18:38:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 18:38:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 18:38:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 18:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 18:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 18:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 18:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 18:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 18:38:49|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 18:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 18:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 18:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 18:38:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 18:38:49|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 18:38:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 18:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 18:38:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 18:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 18:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 18:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 18:38:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 18:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 18:38:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 18:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 18:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 18:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.14 s +[02-20 18:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 18:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 18:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 18:38:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 18:38:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 18:38:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 18:38:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 18:39:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 18:39:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 18:39:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.32 s +[02-20 18:39:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 18:39:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 18:39:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/config.yaml +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 18:39:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the man maintains his serious expression, looking directly at the camera. His attire and accessories remain unchanged: a black durag with strings hanging down, a gray long-sleeved sweatshirt, and a blue sleeveless vest. The camera angle provides a clear view of his face and upper body, with the institutional setting in the background remaining consistent. The lighting is soft, highlighting his facial features and the details of his clothing.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.json +[02-20 18:39:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 18:39:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.mp4 +[02-20 18:39:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 18:39:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 18:39:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 18:39:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 18:53:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 18:53:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 18:53:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 18:53:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 18:53:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 18:53:52|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..6166674980a44770d10311f15a62bed38d45d887 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up shot of a man's hands as he holds and points to a legal document titled 'AGREEMENT BETWEEN BRISTOL & AVON CONSTABULARY AND CHRISTIAN TAYLOR.' The document is marked 'RESTRICTED' and contains detailed text about an agreement involving cooperation in police investigations. The man, wearing a gray long-sleeved sweatshirt under a blue sleeveless vest, uses a black pen to highlight specific sections of the agreement. The camera is initially focused on the document and his hands, capturing the texture of the paper and the details of the text.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a50c3fd46115f9f9842825fbad2acb489c9e6880 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4a00a40066b47bb585278a2712ac3a1db8a54208919388d06b393d6f7e5d7bf +size 2179898 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..0843325733028f3f7e3d2fcd801c8789f672f9ca --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera begins to pull back, more of the man's upper body becomes visible. He is seated at a table, and his attire is revealed: a gray long-sleeved sweatshirt under a blue sleeveless vest, and a black durag with strings hanging down. His hands continue to hold the document and the black pen, and he appears to be carefully examining the text. The background starts to come into view, showing a light-colored wall with a window and some posters or notices on the wall.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4808fb632122d9e68a4187d0a1d1dfd9a2cfadad --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc98dbdc77fc36bfaefb4401556a1d4acda3d425d8a5c672d7e696bad3430ae0 +size 1299889 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..17525a7b751a698809d524fa013eb147257beef3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera continues to pull back, revealing the man's full upper body and part of his face. He is seated in what appears to be an institutional setting, possibly a police station or interrogation room, with light-colored walls, a window, and some posters or notices on the wall in the background. His attire remains consistent: a gray long-sleeved sweatshirt under a blue sleeveless vest, and a black durag with strings hanging down. He holds the black pen in his right hand, resting it on the document, and maintains a focused gaze towards the camera.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..963849a6707656d4ef5e4b8585c959be6e47f555 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c33795bd9cfc7cfcdb5b9b9e60e4db657c31e7db590d0dccdb691f3a5fbb90 +size 752379 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..117ce1e599368abd520d4e11432ac445e5b55723 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The camera now focuses on the man's face as it pulls back further. He is wearing a black durag with strings hanging down, a gray long-sleeved sweatshirt, and a blue sleeveless vest. His facial features, including his beard and mustache, are clearly visible. He looks directly at the camera with a serious expression, occasionally glancing down at the document in front of him. The background remains consistent, showing the institutional setting with light-colored walls, a window, and some posters or notices on the wall.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7913d6d8a14e1493257af48dd63ebbc7f8c0d7e3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:374ad4c2d0235b0b773af23f7d0237aa458735a264889df6e16a01bc072a3bbc +size 996922 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..628d511f30614e6ca8ccc77a345f72a354d47c58 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the man maintains his serious expression, looking directly at the camera. His attire and accessories remain unchanged: a black durag with strings hanging down, a gray long-sleeved sweatshirt, and a blue sleeveless vest. The camera angle provides a clear view of his face and upper body, with the institutional setting in the background remaining consistent. The lighting is soft, highlighting his facial features and the details of his clothing.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..35eba3ae0d4a558c693bf00a226f94029108fe22 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6ff392fbeba43b90c529e1f9b2e26c3c483d347ca8114197cbc196a8993a5e +size 861766 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fa7c46fb07145c9e2bbeb053ce7e087a811add0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c777e6ff932c0e41a2439203a9b42dd12b67b910 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/console.log @@ -0,0 +1,22 @@ +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/config.yaml +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.json +[02-19 20:06:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:20:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 20:20:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 20:20:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 20:20:30|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c4a9c7b7d48b7f8c8d4864f2646b8ef29a440ab3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/debug.log @@ -0,0 +1,352 @@ +[02-19 20:05:18|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 20:05:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:05:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:05:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 20:05:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 20:05:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 20:05:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 20:05:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 20:05:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 20:05:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 20:05:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 20:05:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 20:05:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 20:05:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 20:05:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 20:05:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 20:05:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 20:05:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 20:05:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 20:05:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 20:05:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 20:05:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 20:05:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 20:05:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 20:05:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 20:05:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 20:05:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 20:05:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 20:05:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 20:05:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 20:05:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 20:05:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 20:05:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 20:05:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 20:05:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 20:05:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 20:05:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 20:05:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:05:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:05:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 20:05:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:05:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 20:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:05:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 20:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:05:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 20:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-19 20:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 20:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 20:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 20:05:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:05:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:05:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:05:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:06:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 20:06:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 20:06:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.50 s +[02-19 20:06:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:06:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:06:06|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/config.yaml +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 20:06:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The scene begins with a medium shot of a man and a woman standing in a dimly lit office space, bathed in blue light. The man, with short dark hair, is dressed in a black jacket over a black shirt, with a lanyard and ID badge around his neck. The woman, with short brown hair, wears a black suit jacket over a white collared shirt, also featuring a lanyard and ID badge. They face each other, seemingly in a serious conversation. The camera remains steady, focusing on their upper bodies. In the next segment, the man subtly shifts his gaze and then reaches off-screen to pick up a disposable coffee cup with a brown sleeve and black lid. He holds it in his right hand as they continue their interaction, maintaining eye contact.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.json +[02-19 20:06:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 20:06:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 20:06:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 20:06:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 20:20:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 20:20:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 20:20:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 20:20:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 20:20:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 20:20:30|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ddb679fa06587c97e0661356a8a29e1f09c335ef --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The scene begins with a medium shot of a man and a woman standing in a dimly lit office space, bathed in blue light. The man, with short dark hair, is dressed in a black jacket over a black shirt, with a lanyard and ID badge around his neck. The woman, with short brown hair, wears a black suit jacket over a white collared shirt, also featuring a lanyard and ID badge. They face each other, seemingly in a serious conversation. The camera remains steady, focusing on their upper bodies. In the next segment, the man subtly shifts his gaze and then reaches off-screen to pick up a disposable coffee cup with a brown sleeve and black lid. He holds it in his right hand as they continue their interaction, maintaining eye contact.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..295a8e09e50db9ae9b5c6d31027617082c612870 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4237a7af9eac8f486df38376325f7942ef78198babc35eac2889506c89a7f523 +size 410839 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5ba1033f05ec77a9677c2373a945d6b364e5ae3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..4d9464d838a4e889b897b292e720a575416bc6fb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/console.log @@ -0,0 +1,22 @@ +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/config.yaml +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.json +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:11:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:11:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:11:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:11:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..18a782a3f832d5ccc2037d8f7a2f42c0d16f1e23 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/debug.log @@ -0,0 +1,353 @@ +[02-20 12:57:43|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 12:57:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:57:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 12:57:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 12:57:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 12:57:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 12:57:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 12:57:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 12:57:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 12:57:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 12:57:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 12:57:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 12:57:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 12:57:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 12:57:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 12:57:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 12:57:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 12:57:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 12:57:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 12:57:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 12:57:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 12:57:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 12:57:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 12:57:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 12:57:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 12:57:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 12:57:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 12:57:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 12:57:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 12:57:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 12:57:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 12:57:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 12:57:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 12:57:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 12:57:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 12:57:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 12:57:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 12:57:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 12:57:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 12:57:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 12:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 12:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:57:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 12:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 12:57:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 12:57:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 12:57:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:57:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-20 12:57:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 12:57:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 12:57:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 12:57:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 12:57:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 12:57:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 12:58:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 12:58:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 12:58:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.79 s +[02-20 12:58:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 12:58:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 12:58:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/config.yaml +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 12:58:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final moments of the video, the man in the green suit keeps the handgun pointed slightly forward, his right hand steady. The woman in the pink top continues to lean against him, her head near his shoulder, indicating a sense of trust or reliance. The man's expression remains serious, and he appears to be speaking to the woman, who listens intently. The background remains consistent, with the striped armchair and light-colored wall providing a stable setting. The camera stays focused on the pair, capturing the culmination of their interaction." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.json +[02-20 12:58:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 12:58:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.mp4 +[02-20 12:58:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 12:58:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 12:58:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 12:58:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 13:11:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 13:11:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:11:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:11:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:11:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 13:11:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..eca673f6a8fd13020ae21634f6efc3d611e333c1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of a man in a green pinstripe suit jacket and white shirt, seated on a striped armchair. He has short black hair and wears a silver wristwatch on his left wrist. In his right hand, he holds a silver-colored handgun, which he examines closely. A woman with curly dark hair, wearing a sleeveless pink top, leans against him, her head near his shoulder as if whispering to him. The background features a light-colored wall and the armchair's vertical stripes. The camera remains steady, focusing on the man's hands and the woman's proximity.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6848e2ab9a65be9b26be255cc847fac447f51cd1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a874b29eb7577bc0098c96e2472aa6326f30ccf5dff18fd6ea11de931527f62c +size 637144 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..e45af6bcf3a9d0d6a8b06f79db97339c457d88f9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the man in the green suit continues to manipulate the handgun in his right hand, his fingers adjusting its position. His gaze remains fixed on the weapon, and his expression is calm but focused. The woman beside him remains close, her head still near his shoulder, suggesting an intimate or tense moment. The camera angle stays consistent, capturing the subtle movements of the man's hands and the woman's slight shifts in posture.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6997ef8a05319badf37cc4156356f9fab13e5889 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:347908a234b7ddeecfe9440a7f60b751aa9201a2c3dc716bd62aa46f33c59e80 +size 874945 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4b913bb6eb68dfb3f1c029adfad094e9bbce804d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The man in the green suit raises the handgun slightly, his right hand gripping it firmly. His eyes shift from the gun to the woman beside him, who continues to lean against him. The man's expression becomes more serious, and he appears to be speaking to the woman, possibly giving instructions. The woman listens intently, her head still near his shoulder. The background remains unchanged, with the striped armchair and light-colored wall providing a stable setting.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7148bec0484c6e3949a8a8a944ba26492044e55e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0160e2611296df8678412ce390e29ab27ff8b98fa5d833720fdb286439587f22 +size 824505 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b59dff9d6ec10e618b2d02a6de3cefb9dc7a06ed --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The man in the green suit maintains his grip on the handgun, now pointing it slightly forward. His expression is firm, and he continues to communicate with the woman beside him, who remains close, her head near his shoulder. The woman listens attentively, her body language suggesting a deep connection or dependency. The camera remains focused on the pair, emphasizing the intensity of the moment.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c00d7898c59a329f63bc5705a48cf4acad5f96b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48e2db62c26f840ebb8787ac64e010467c896e45463e4454a273935585a2cc2c +size 635691 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..bda6261bfe5001d29760d433671b4e914d841346 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final moments of the video, the man in the green suit keeps the handgun pointed slightly forward, his right hand steady. The woman in the pink top continues to lean against him, her head near his shoulder, indicating a sense of trust or reliance. The man's expression remains serious, and he appears to be speaking to the woman, who listens intently. The background remains consistent, with the striped armchair and light-colored wall providing a stable setting. The camera stays focused on the pair, capturing the culmination of their interaction.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..380cb058b8d8923968e8fa0fce0bdc3e3c709fb4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e39dcca0196d483a69ec930ffb0478d61ef59e7ed4b0f4aae320de3a857ac0ce +size 551607 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e81995c017f229af52efa954448ee6eb123fdb17 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/console.log new file mode 100644 index 0000000000000000000000000000000000000000..6ec1fccbf47f4352f94c6e848876dbbee7dc5e00 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/console.log @@ -0,0 +1,22 @@ +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:13:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:13:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/config.yaml +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.json +[02-20 13:13:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:27:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:27:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..31f6242ab972be9f114c8ac77fd28c4d67809106 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/debug.log @@ -0,0 +1,353 @@ +[02-20 13:12:56|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 13:13:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:13:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 13:13:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 13:13:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 13:13:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 13:13:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 13:13:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 13:13:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 13:13:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 13:13:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 13:13:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 13:13:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 13:13:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 13:13:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 13:13:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 13:13:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 13:13:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 13:13:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 13:13:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 13:13:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 13:13:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 13:13:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 13:13:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 13:13:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 13:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 13:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 13:13:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 13:13:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 13:13:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 13:13:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 13:13:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:13:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 13:13:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:13:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 13:13:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 13:13:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 13:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 13:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-20 13:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 13:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 13:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 13:13:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 13:13:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 13:13:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 13:13:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 13:13:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 13:13:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.44 s +[02-20 13:13:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 13:13:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 13:13:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 13:13:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/config.yaml +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 13:13:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the man remains seated on the edge of the well, looking down thoughtfully. His body language suggests he is reflecting or waiting. The camera remains steady, capturing the serene and contemplative mood of the scene. The stone well and red wall with intricate designs continue to frame the shot, emphasizing the peaceful atmosphere.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.json +[02-20 13:13:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.mp4 +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 13:13:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 13:13:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 13:13:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 13:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 13:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 13:27:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 13:27:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 13:27:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 13:27:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..1f788c3d327555e62a9823b7c6376ba34317302e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with two men in traditional attire standing near a stone well in a courtyard. The man on the left wears white gloves on his left hand and holds a small bundle of sticks, while the man on the right has no gloves and appears to be listening. Both are dressed in blue shirts, gray vests, black pants, and black shoes. They face each other, seemingly engaged in a quiet conversation. The background features a red wall with ornate gray and gold patterns, and the well is made of weathered stone.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cd9b7316e501968bfbb214ca69c70fec3cab601 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbde8f763ebbbc617472194994f116dddb4cdbd18dc82a4da738ec94ef2d6bc0 +size 784117 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..90825e995dae1afb82d0c839df3e3a029662a3b8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the conversation continues, the man without gloves gestures slightly with his free hand while speaking. The man holding the bundle listens attentively, nodding occasionally. The setting remains the same, with the stone well and red wall providing a static backdrop. The camera remains focused on the two men, capturing their subtle movements and expressions.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c70c57f690e449385c0d81a26912881dfcd13e7d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b7f0ff35778b8e2180883e23d8c2e9f5dc28b581ef1e7b7e62ac94b4afe3306 +size 929769 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e0accb24f806a0498bc388485a9e4e849fb5cb24 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The man without gloves then turns and walks out of the frame to the right, leaving the other man alone by the well. The man who remains looks down briefly, then begins to move toward the well's edge. The camera follows his movement slightly, maintaining focus on him as he approaches the well.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8cd0dda0e2166598026cfc87820f251844e2c3c0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9ad20ed5eeab5ad31a76e95bf301090c08aa906282e7e82292353b42ac51a38 +size 1007732 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..72ded24d5b3ea76f234525fa92d484dc473d3868 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The man sits down on the edge of the well, his posture relaxed but contemplative. He looks down at the ground, his hands resting on his lap. The white glove on his left hand is still visible, and his blue shirt and gray vest remain unchanged. The background, including the red wall and decorative patterns, stays consistent throughout.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1226b75fb43ca60dfc5232f94a7a997e99f6b5d4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cec4f40bef41e9d97ae68bd76ab40473edd5b249b255d45c9e7ab1f9ec5be76 +size 857696 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..b24964ca76eb0b2efa1217f1935a7052a3a6529d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the man remains seated on the edge of the well, looking down thoughtfully. His body language suggests he is reflecting or waiting. The camera remains steady, capturing the serene and contemplative mood of the scene. The stone well and red wall with intricate designs continue to frame the shot, emphasizing the peaceful atmosphere.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e20da7965106a230d04aa9e6df57b12b2309d678 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:165d02dd53d5167097c11dd5c40b3a1143e78daa6fafc4334d9ad264b90c5bcb +size 439912 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d67ba034a440020b34c89d562d5e180bbac838bb --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..0ea320a4c6a8a18d732391b782d126db9dd96e17 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/console.log @@ -0,0 +1,22 @@ +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/config.yaml +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.json +[02-19 20:47:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:01:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:01:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:01:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:01:12|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9bd17083429a93b7c5720ea535b617e027ee6d26 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/debug.log @@ -0,0 +1,353 @@ +[02-19 20:47:04|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 20:47:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 20:47:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 20:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 20:47:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 20:47:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 20:47:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 20:47:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 20:47:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 20:47:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 20:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 20:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 20:47:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 20:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 20:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 20:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 20:47:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 20:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 20:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 20:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 20:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 20:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 20:47:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 20:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 20:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 20:47:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 20:47:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 20:47:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 20:47:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 20:47:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 20:47:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 20:47:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 20:47:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 20:47:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 20:47:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:47:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:47:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 20:47:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:47:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 20:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:47:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 20:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:47:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 20:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-19 20:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 20:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 20:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 20:47:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.15 s +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.68 s +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.68 s +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.68 s +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:47:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:47:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 20:47:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 20:47:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.89 s +[02-19 20:47:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:47:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:47:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/config.yaml +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:47:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Finally, the two men stand apart but still connected by the shared moment. The man on the left looks at the camera briefly, while the man on the right maintains his stance near the truck. The background remains consistent, with the silver pickup truck and green trees framing their interaction, capturing the emotional weight of their farewell.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.json +[02-19 20:47:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.mp4 +[02-19 20:47:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 20:47:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 20:47:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 20:47:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 21:01:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 21:01:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:01:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:01:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:01:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 21:01:12|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..69b15c250ce9bd906c4732741e14eeffefc2e69b --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with two men positioned at the back of a silver pickup truck, set against a backdrop of green trees. The man on the left, sporting dark hair and a short beard, is dressed in a light gray and dark gray horizontally striped polo shirt, dark pants, and a black belt. The man on the right, with white/gray hair, wears a blue and white checkered long-sleeve shirt, blue jeans, and a brown belt, and holds a pink object in his left hand. Both men are initially facing away from the camera, then turn to face each other, beginning a conversation.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..605ef0f5844f8800b0d3c846df0d9601ca558a72 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:139793233c0cce1a92f3be8d896f3f3e577d5b8571f8389cad033d51b4399903 +size 923123 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5c89e78fab9d15da606baad4ebdb10db6b79d16d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the two men continue their interaction near the truck. The man on the left places his hand on his hip while listening to the other man speak. The man on the right gestures slightly with his free hand, emphasizing his words. Their body language suggests a friendly and relaxed exchange, with both men smiling and nodding occasionally.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2c0d6336c88d17a1534c72b11c0957313bc8bf06 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d1e4580201f7ea9b2b9e515e559f802f254339d2ed1e05db32bc36f6bd057b2 +size 1174631 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..7bb3dd191f2b5d68f73ad60c3a55b9ee289a1c9a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The interaction evolves as the two men shake hands, symbolizing agreement or friendship. Following the handshake, they share a warm embrace, indicating a deeper emotional connection. The man on the right, still holding the pink object, leans into the hug, while the man on the left reciprocates the gesture, showing mutual respect and affection.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a230d8b5fce0d06fc4609a219cf69d2e9166a4d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9e5d695a05943b9e30c19d8298e1876fe5dbe7fd7db76dc737012cc3d610095 +size 1284966 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..244cff1ddcde70fc2c7e006dd74efd7ab42b40d6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"After the embrace, the two men step back slightly, maintaining their positions near the truck. The man on the left turns slightly toward the camera, while the man on the right continues to hold the pink object. They appear to be concluding their interaction, possibly exchanging a few more words or nods, signifying the end of their meeting.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9527c51f9b3e608a5f112ddcadb599c0081d440d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56abd5cd7001fddcdfe8854861031c64243713495ed7fb4e50add703bd6ec038 +size 1258142 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..4c528bee8078a3a5802c04b7d3d9eb8e51eb7c8f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the two men stand apart but still connected by the shared moment. The man on the left looks at the camera briefly, while the man on the right maintains his stance near the truck. The background remains consistent, with the silver pickup truck and green trees framing their interaction, capturing the emotional weight of their farewell.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ae8484d0d7661104ae34a56d5f7259550dbf8242 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:225f8607246352a1f2de376baf70b792e93dce870607a1d0c558c6731d548fe8 +size 885969 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25064fd0832d62919eed55f9a5cad50c3d7634ac --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/console.log new file mode 100644 index 0000000000000000000000000000000000000000..be654349e2e23b43eb0ad6981f442ffc622b5e97 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/console.log @@ -0,0 +1,22 @@ +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/config.yaml +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.json +[02-19 20:48:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 21:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:01:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:01:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b7b8716f3ee35d21ed69f5aa045a8d7a41495d61 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/debug.log @@ -0,0 +1,353 @@ +[02-19 20:47:35|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 20:47:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 20:47:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 20:47:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 20:47:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 20:47:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 20:47:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 20:47:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 20:47:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 20:47:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 20:47:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 20:47:45|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 20:47:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 20:47:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 20:47:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 20:47:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 20:47:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 20:47:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 20:47:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 20:47:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 20:47:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 20:47:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 20:47:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 20:47:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 20:47:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 20:47:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 20:47:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 20:47:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 20:47:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 20:47:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 20:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 20:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:47:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 20:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 20:47:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.05 s +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 20:47:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.66 s +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.66 s +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.66 s +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 20:47:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 20:47:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 20:48:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 20:48:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 20:48:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.70 s +[02-19 20:48:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 20:48:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 20:48:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 20:48:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/config.yaml +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 20:48:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the camera captures a sweeping view of the entire stage, emphasizing the scale and grandeur of the performance. The students are seen in their full formation, with the clarinet players in the front row and other band members behind them. The conductor leads the ensemble with confidence, and the audience fills the theater, creating a lively and engaging atmosphere. The video concludes with a final shot of the stage, highlighting the students' dedication and the festive spirit of the event." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.json +[02-19 20:48:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 20:48:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.mp4 +[02-19 20:48:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 20:48:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 20:48:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 20:48:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 21:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 21:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 21:01:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 21:01:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 21:01:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 21:01:50|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8fc591915ae2ec3c2b129169d2784ea484cd24be --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of students in school uniforms, focusing on their upper bodies as they play clarinets. The students have long dark hair and are dressed in white collared shirts, dark blue vests with white buttons, and red and white striped ties. Their hands move rhythmically over the black and silver clarinet instruments, and their facial expressions convey concentration and enjoyment. The camera captures the intricate details of their uniforms and the gleam of their instruments under the stage lights.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7862b0715da0919be39b3631bf545e95c87ce370 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cacc830cbb454c0bc8622f3c9f5f69b527985d3590ca22b5ce7ab35e6503301 +size 1451010 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..fe0ad8819a5afb67ecf95281e978df81dde772ea --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the camera slowly pans to the right, revealing more students joining the performance. Each student holds their clarinet with precision, their fingers deftly pressing the keys. The background remains consistent, with the colorful stage decorations and the large screen displaying Japanese text and illustrations. The students' synchronized movements and the harmonious sound of their instruments create a captivating visual and auditory experience.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0b06ca6005832ad953c7d52c56cc7a1812dc35a4 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c8925ea3fe9f3d48f7e43db79af57ce3428703c77336757aa1b63f25d14b6ad +size 1404389 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..f84f63634b473a5f88a6249e685bc9326fb265c9 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera continues to pan, showing the full extent of the stage and the arrangement of the band. The students are now seen in their complete formation, with the clarinet players in the front row and other band members behind them. The conductor stands at the center, guiding the ensemble with clear and expressive gestures. The stage is brightly lit, and the colorful backdrop enhances the festive atmosphere. The audience is visible in the foreground, seated and watching the performance with interest.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..50f73c23a833670fcf464aad04068135550212ec --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b572428d5b384e5eb310ab27a127a4c9c60715d70d9a3a7322cce3d70ca0042a +size 1707415 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..94d39d8864240f12261f5eec96146e47c877f990 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The video then zooms out further, providing a wider view of the stage and the audience. The students' uniforms and instruments remain consistent, with the dark blue vests and red ties standing out against the bright stage lighting. The large screen in the background displays the event's title in Japanese, surrounded by playful cartoon characters and vibrant colors. The students' coordinated movements and the harmonious music create a sense of unity and excitement.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49de781b99781c26942df883980b031bb7e900c8 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:725d2192a9bb4714165defd5fa19c0185974044bff7c5f4a0aa101777fce2097 +size 2295341 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..72f027a5c1739c7b0837360334eccb5b870862a3 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the camera captures a sweeping view of the entire stage, emphasizing the scale and grandeur of the performance. The students are seen in their full formation, with the clarinet players in the front row and other band members behind them. The conductor leads the ensemble with confidence, and the audience fills the theater, creating a lively and engaging atmosphere. The video concludes with a final shot of the stage, highlighting the students' dedication and the festive spirit of the event.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..329e664e743ae718449420c8b179e862087d775f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc3190e63398628bb0e7be1dfb7147b33d02d9ec196e4076978d86fe4e19b310 +size 1896203 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15d2b2a6c3e34bf8c9deb523f9c3be00fdc9b67d --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/console.log new file mode 100644 index 0000000000000000000000000000000000000000..a43b1ea78f0ae830909aaa13f0f6c4a41271769f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/console.log @@ -0,0 +1,22 @@ +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:54:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:54:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/config.yaml +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.json +[02-20 08:54:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 09:09:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 09:09:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 09:09:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 09:09:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e6f6beee52d36cd27e143f734843a0bd086532a0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/debug.log @@ -0,0 +1,352 @@ +[02-20 08:54:08|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 08:54:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:54:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 08:54:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 08:54:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 08:54:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 08:54:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 08:54:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 08:54:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 08:54:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 08:54:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 08:54:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 08:54:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 08:54:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 08:54:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 08:54:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 08:54:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 08:54:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 08:54:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 08:54:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 08:54:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 08:54:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 08:54:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 08:54:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 08:54:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 08:54:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 08:54:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 08:54:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 08:54:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 08:54:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 08:54:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 08:54:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 08:54:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 08:54:20|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 08:54:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 08:54:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 08:54:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 08:54:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 08:54:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 08:54:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 08:54:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:54:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 08:54:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:54:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 08:54:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 08:54:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 08:54:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 08:54:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 08:54:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 08:54:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 08:54:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 08:54:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.29 s +[02-20 08:54:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 08:54:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 08:54:55|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 08:54:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/config.yaml +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-20 08:54:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video starts with a man in a gray short-sleeved button-up shirt and dark hair standing near an open car door on a bustling city street. He is accompanied by a woman wearing a colorful patterned jacket (blue, green, yellow) over a yellow top, with long dark hair and a large gold-colored shoulder bag. In the background, a third person in a black outfit is partially visible, along with storefronts, including one with a red circular sign and another displaying a menu board. The man and woman appear to be engaged in a conversation, with the woman gesturing slightly as they stand close to each other. As the scene develops, they move closer and share a warm embrace, conveying a sense of affection or comfort. After the embrace, they separate slightly but continue their interaction near the car, maintaining eye contact and body language that suggests an ongoing, meaningful exchange.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.json +[02-20 08:54:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 08:54:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 08:54:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 08:54:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 09:09:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 09:09:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 09:09:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 09:09:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 09:09:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 09:09:20|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ee4dace81e83df935e078c84fb1e5ca303d575d6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a man in a gray short-sleeved button-up shirt and dark hair standing near an open car door on a bustling city street. He is accompanied by a woman wearing a colorful patterned jacket (blue, green, yellow) over a yellow top, with long dark hair and a large gold-colored shoulder bag. In the background, a third person in a black outfit is partially visible, along with storefronts, including one with a red circular sign and another displaying a menu board. The man and woman appear to be engaged in a conversation, with the woman gesturing slightly as they stand close to each other. As the scene develops, they move closer and share a warm embrace, conveying a sense of affection or comfort. After the embrace, they separate slightly but continue their interaction near the car, maintaining eye contact and body language that suggests an ongoing, meaningful exchange.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63117c457a42b572b7d160d6fb317018d2d2c670 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7701c9f9b79096984380edc245e8798e50c03e179adbe41867367dcd4554d81a +size 1178370 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/config.yaml b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2616e16184a92e1aef46a1ee683dd4b72f0fe88 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/console.log b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b9761975619d266eab135351618aa4ed8523701a --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/console.log @@ -0,0 +1,22 @@ +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:11:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:11:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/config.yaml +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.json +[02-20 00:12:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:26:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:26:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:26:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:26:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/debug.log b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d7055888f4a98b3df651da26bd18e1387f3e4e38 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/debug.log @@ -0,0 +1,353 @@ +[02-20 00:11:16|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-20 00:11:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:11:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-20 00:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-20 00:11:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-20 00:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-20 00:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-20 00:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-20 00:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-20 00:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-20 00:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-20 00:11:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-20 00:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-20 00:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-20 00:11:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-20 00:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-20 00:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-20 00:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-20 00:11:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-20 00:11:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-20 00:11:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-20 00:11:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-20 00:11:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-20 00:11:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-20 00:11:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-20 00:11:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-20 00:11:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-20 00:11:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-20 00:11:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-20 00:11:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-20 00:11:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-20 00:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-20 00:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:11:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-20 00:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-20 00:11:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-20 00:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-20 00:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.08 s +[02-20 00:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-20 00:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-20 00:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-20 00:11:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-20 00:11:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-20 00:11:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-20 00:11:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-20 00:11:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-20 00:11:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-20 00:11:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-20 00:11:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-20 00:11:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.33 s +[02-20 00:11:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-20 00:12:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-20 00:12:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-20 00:12:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/config.yaml +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-20 00:12:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Finally, the video concludes with the man on the right continuing his animated expressions and gestures, his energy undiminished. The man on the left remains completely motionless, his identity and appearance consistent throughout. The ship's sail and distant crew members provide a stable background, ensuring the focus remains on the two central figures and their contrasting dynamics." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.json +[02-20 00:12:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-20 00:12:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.mp4 +[02-20 00:12:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-20 00:12:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-20 00:12:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-20 00:12:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-20 00:26:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-20 00:26:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-20 00:26:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-20 00:26:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-20 00:26:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-20 00:26:29|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.mp4 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_0.json b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2b1fe8616d0efda6b83d92217607d5bd23a392ab --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a static shot of two men on a ship's deck under a bright blue sky. The man on the left, with dark hair tied by a white headband, wears a black robe and gray scarf, standing still with a serious expression. To his right, a larger man with a purple mohawk, red face paint, and a studded brown vest begins to make subtle facial expressions and slight head movements, hinting at an internal reaction or conversation.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_0.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..596c7a365b86114c23c9e1124911ebc048262dfe --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a65cb7245a2eb4f3e89093293cb105441c7b269feefd0a1a3cd1c7b4a126a88 +size 705098 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_1.json b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..bd834e263846a51bf8052c81a0dbfc965c6f0100 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the scene progresses, the man on the right becomes more expressive, his face contorting into various emotions—furrowed brows, raised eyebrows, and slight smiles—as he gestures subtly with his hands. His body language suggests engagement with something unseen, possibly reacting to an event or dialogue. The man on the left remains completely still, his gaze fixed forward, creating a stark contrast in their demeanor.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_1.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c9ff03adcd3755079729f400f663a4ff7dae2a8f --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7baecc195f67e6cad68fe65bc7ef765a186d1f818bfc9a329f9616a6f67a26d +size 1081496 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_2.json b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..7181664982747978bd87d39b75c5534a09baadc6 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The man on the right continues his animated display, now incorporating more pronounced hand gestures and head tilts, further emphasizing his dynamic presence. His facial expressions shift rapidly, conveying a range of emotions. Meanwhile, the man on the left remains unchanging, his posture and expression frozen, highlighting the tension between their contrasting states.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_2.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33d9c204c45cf2d3359e85a1d9e2836af8f601f1 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75f92eef147e991b514a0347e6fd0bddcd58f66c49758c3bba7e67a383d1ee6b +size 1419860 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.json b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..d0be326b7293fec373ae8ed5ce09acdcf6360e9e --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The interaction between the two men unfolds as the man on the right maintains his expressive behavior, his movements becoming slightly more exaggerated. His red face paint and purple mohawk stand out vividly against the backdrop of the ship's sail and crew. The man on the left, however, remains steadfast, his stillness serving as a counterpoint to the other man's liveliness.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29cead31fa33300fd49dbed2fd0366e5ea403bcf --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b81346152a2afaece10a26c2b869dcf27a782a128e9cb8bee93e1a439594e4a +size 1147292 diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.json b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..6f388b2ab8deb8f44033a663fdc729d30bcd1588 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Finally, the video concludes with the man on the right continuing his animated expressions and gestures, his energy undiminished. The man on the left remains completely motionless, his identity and appearance consistent throughout. The ship's sail and distant crew members provide a stable background, ensuring the focus remains on the two central figures and their contrasting dynamics.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.mp4 b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6c6d6b2ee704d36f9c57aa7f8b3b21d936d3a0d0 --- /dev/null +++ b/0219194937_cosmos_minibench/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a9e03879af1f254be05ab2e3707bb4c57f6df2135e0f7a0bb5f21230458aa9a +size 1174537 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1aa3eff0266d2b8d89db35176c44a6e757f7c3a6 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_02b6db2cef4b6a8613d16387fdd0b9d57b6c9f11e1d88200bb0772a5a6722540.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1539efd7155d9ee964cf9252451765633b96db250cdacb2d3b342505eb2e29ad +size 1535084 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bcb8e7c3615a5488c29674b9775623c5cb22df97 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_0aa5ec6e4732891d53de496de0ae8e99a78e2782a13a4b0606635441bcb3cdf4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5373daec9fd12b26ceb12ce411bc0b8609625a14652454672457067801ef123 +size 868229 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4b765c8d0da9b99c00b199d6347b8ccdfa14a55 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_16c80546a74a64f44e2fef9d4e7b58c1da1f0eae000559a5370988c1e15c487f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3406fb5ea2009d22e628d84c60def673a1ceb2be3c7a3291ded8ba1cdaa8383 +size 446811 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38626929474e20619ec06d79fe980074ca5738b8 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_1c4ba82450b80576babd5138c9b16a79dc7eb344e6871f04152a0f8765e545dc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5090f0b294571b5580083d20f881beb2cfdbc4696641a5a27b826e38e33c5dd4 +size 892807 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39db7f89c759a5a11b0768438f840b6839db898b --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_26961061cf21f9054ea3b1e8088d0e3a82efe6b20f28527d3ed9d0b7b7ea52bb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30467c28eeaa6b40383d2b92bdc2a226278d3893cffbb1aff6b736425c75c6bc +size 6181741 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..490d631d0226f2464153c4908c69436f23750736 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_2c00f081fea65208fe8caabbcb639cf5eb1a993cd1ea7a8b0657c4e28fce7d7f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17377b8717c462edaad7e018f3963f5fc90b55a34d5961ba373b00f1042cba5c +size 848901 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..994af532b6fe6bb84ec875a8c358eb3453bf38ce --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_34a9cf34367e122aa5bb4d4025a727eb2673f255a962ac9418a3f822219fc81a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa9ef41458b0a89cccf9239861cde2f71ac3992a574a0a5eb65a237425cf24a +size 755054 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48798895900d1b8ba6c2afa6d3f1ea0fd600568f --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_3d540f5d803bf7a4b5217ff2d852ca1d207fdd6f2434b3022ef600e9ba743b25.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c240544c6438023029812de720bfdecf854a736af12805cc74848a2cf5779a01 +size 531782 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8db2c49f53ccb86b508a29b4fa50ba45fcdec052 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_4bef8bf072708642efd68a64bd93c35079c23e12154cf6497eb57796271b1e7e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:428c28a2ac630e488ca018449d698dc92adb5cd84f4bd3ed2652b9e9ee927cbf +size 476117 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..80295346f6e1d54cf245811990e2aa0a93b3e112 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_56d5b4dbc39f8dc1855e535a9fbd6806afa044224b98fe909dc19e3707f9f6bc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:327c48150465951f6a50cb46303463947fd380bc36b004daa3b53cc4c033daae +size 1196871 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05558f1e86256fb195a363a1e00df33d9dba6a8f --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_5adf4e81b0f7f75936ccabacd27772f5321649ec52c41c5cc2fcb9158aed7b8d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68c2612269f0598991ff46ac7d430852d0886dad6b00509ff34d7384a7209ebc +size 1581122 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..993eee8736cda07b800812921f20a6f1d01ede6f --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_6e55f7c593783770206ef013848e0e9dafbc9f1ea365ec400dc5129acd36408f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d52725d69da51f1bbc6e728a1f49811b697db7ee31fd324d11c8fc4ab81ea64 +size 2493529 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6b6cb7f06b1990adb2d5fb77d8725155fbd473d3 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_6f1d4570449bb23644e2d1cbfc45e7b5554faec9978a326cf2b0ec009f875325.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6941eac3919eaba22cae3944c207a9c6dbcc5f190720a9dac27f8babb6422ad +size 571684 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8abd7875358c017929fa8a2dc639e886a6e7538f --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_72a2b27fd45882b4c67384c5a0a7ce24eec38bad05da4cfdccbafb3e60535a89.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac927d2bb683d276dd0e1496bef2f407e3833a230aee920a362c143f9c64ef44 +size 1405768 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..afeb593c1a74e96de4ded01e2ccecd3c2a8327d0 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_74c0294086072e99e31130225d7947b69859b08a3b1e4c1c85b4d1c11a674f45.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:800d080f65ef512498cb603bbe56dfbd35b745d4e72957039a166454d79edaf0 +size 1435766 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b25bbe9290761ea0ba0a7ba306536db4990c358 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_7615787f996e20fcf9eae1edb66fe0402ee0ec851c0c47e4d3788e4355565c6f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2f6a28c5e1306a3d36a0f45d5eeb6c8901bb64812768b34795ad33839417db +size 851825 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f769c75d203402f7f09a512b601d92f9a58e1def --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_773d17b8da2db6d05e0dba41287740a5b262305dbb9adb404cdf6b96052e8931.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:183c6905dbe9769324e3b3fb25ac1bd98da860d0bfb9ad6aa0feded70a775c42 +size 798234 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..20c5e6fa1f2627afa7a484dbf9b437e91ea111e8 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_774f593ce99ca64ae052bf8b9f0b4eb5df6c6df8f3f607f90c44e89f5efc3c3f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:409d0207c2e6fb7e8b1e9446676346e7212caa4e84a00df75d633afe066af82b +size 935335 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8cdb1bef722c5357d59b3e9a147cffc62995711c --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_77d03a3ab29ed3365fe2ec2efee850a110edd47aa5aaf77c2e24e91c88407562.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e25c54a016f3a3857158768050ae7f1d42a63c074280a0e1a2413c2868fc15fc +size 1176314 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d476f29a684141b959ef5165af74cf2d38e8ad2e --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_7a8d773dcb394a703526730121477ddd52e2b214bc6adc6fc1841cc3790c846b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6aeedbac9d43fe15d8af8d9e62b1d45f6975c7b3f1cb0266047bba2f3897f49 +size 5178427 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dabb4c4a375c65c9fe0b7145dac54e77c1a60cac --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_7e740139d7cedce0245611090627289da4655f713b653db0fdbada748a011f70.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e2818eaab09157399252ecfd4eac30a78d31c81202c9c8186588aa7dcd4dc10 +size 1356992 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..040636c6ff5eeef05027afd64a00cae205b9c8e9 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_863ac656af89795fefd666f839739a4bdc425ce62fdb6126a9f8c372a0c4e175.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf641207d52cad313f94ee8e693312643bb179c3204c11366ac2c059ba71eb35 +size 953569 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d7543faed2641de5dd2c0641325dbdb86bbbef --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_8736ac18f483c27cbde9d332a8299911c682471d52117101f4cff176f2463455.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a792ee2cf9310375d73eff9b378ca2d4200a5fffdc8cf4389c0f493ead84f8b +size 1022301 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..83f9e043f9c5f350bd5203a8b62aa5b254a33d71 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_8a7c5e9dbf69f1926e6e495a53ecaa6db2c53e57b6949dde49d6e65bc01b603a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9161ad596d9ccf2ea33579e68fc03a7c9f1bcac57b91c88d02f54092ad146c5c +size 954388 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..96418edd430bdb85cc7831a7c553897d46f81f9a --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_8abf959e44e15f12eca47f6a46bee424178db17a51ce3e0071f587bdcf4def80.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f5ca18b8bbb16b1b5a30a74e0d0ca9eea77ff8306dc9623dcb0a5ecff667480 +size 6830623 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e39aa914f621b959ce4cc6b94cbdbc178e7fc348 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_93c4b2d6a73244c3f451b0ecb014d7146b96bcfeabf52ede75737c4dbc1368c3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:085d02c50d54423e9713d04779d469a0ac2e02f5551e51fba24b302829dde096 +size 6289870 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62d3178c9f81468ca0e7938e5fd09a9544d05242 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_98082ad25b486f4fab1b7a8b886e83fa006cdf87e1ab16ff1ee0db2de6fedd7b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d68eb6c245565c7c4cdc3034d5c23c17de7e9be084ec23ba8df086305cd654d2 +size 598011 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb4da112e5aeebe063730a35bbc7d922f20de72f --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_9a6cc4d01acff1bc6cc90e3e6c8388172796c9cb9a8de081faf07543ca2d276c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baebb7eea21e9abc17d540a580f8dfb22061bb15a5c005f0dcfb575249de0385 +size 709712 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a4c699f17781fac0266c47a544a39d4b906d17f6 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_a2ea30a78eca4b8a2e1aa47474c91cfe5a13c64fde9329e71b2a52a1034f0bec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28b8bab6c0e8fab46e9b247cb8acea21805b1c15f3c95af085bb887b19272361 +size 599411 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0169aa47b1156303836b2481ee0b81ff7c939529 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_a4e49029992653e0856e6ed16a9ae0b7f86f23e84a5f7100ef9050c42cc9887a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cc096f5b87ab3e75e11d207cd8b2bb08f4716faf9989acd740fd0f58429ef11 +size 769178 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b68a6a10e27bcb7b7a96ab33b6de59d3e665748d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_acdc7abe5754e9b7ebe35ec033786800763261d19afcf20a5f7335ba5ccbdb7e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d24bdcc968bafe317ccd62b3fd2e88d4867dfb58e46d053de97ccc087468fdd +size 1261529 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..86a332764cfa1c7643e8d4556466c80de12d1978 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_b30ce515ae3d463541a74652b902ee64f63a77132bb701b2ad6b52cd4b8549a5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cf166bedd201976545c7e4fd301e191e3cbeec8454f45dca8ca0bad0e2cd07f +size 1816968 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46ab9db63dd5e8d4844d7bebbd14ae102ede6826 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_b3b68dcdac2398b641897eb9c4055d7df221e3835dedc90c4135722887233af2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44262bd852cb41b56a5bfb5ffc51fc40fd03f1aa1427673b989890d8aa962e80 +size 711718 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c4da0622a5e95fe1a3167e0c5b1e20d9eed283c4 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_c9ee58b3a4b17536911bbad080070c70e7d4ddb513eab9a3805aae6035236a89.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5916a79ecace7a626864dd567508630913898bb42305aab0fe9af3e222f63995 +size 791723 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2088d7efa973a812743ad620d8fa22854ee2aae8 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_ccc946b7ca37c3b6266b1875db9abbdc97a8fbe9a3453c902bc21891147d46be.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce185ef5aba3f9692c35fc49b510836c84b2ccd35a5f806ff8f746787558c8f0 +size 1439797 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ffb0e224aa08fd11d8395b21f04aa24599002a07 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_d2498fab7ca118cb774a2ef1169a93b9a5b07b798db4be9045570b07b997ebd0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96996e933aa3c8fd9156d3be83a55a69d9f97586bdb9842247a9ebe3f978e4ce +size 780599 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7c55359759fd59f46b07d6ef0250ea5a4e9451b --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_eaa3fb94fe7a2564d45365c69ec3b88b6f3f9895aa0e3f358627bbf8ead3e847.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c1cc0b4538e79c0567f011beba6acdd6dbc255ce93d1726dca145942221e085 +size 1479486 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..de8c6b3e1e537a488d0aeadec3fbda47c8760971 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_f302a4babcb7181f3f93990920507de832dba3abd5a0fc8503177676a377f709.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d01221098ba7e20506ae5f4acd745bcc56164231759e45ddb4cd5c875cb3ba7 +size 755582 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ec3a93452f3d1c1e72a86282734cc94d4c6b4acd --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_ff18db7264992673349161278b3a9c22ccdfd6c62433641da06fb668768418c2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01e7d508c718826d75b2ce7f6f5f9c519a116f44bd094463049d6a9220ba42e3 +size 6987243 diff --git a/0219194937_cosmos_minibench/videos/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23.mp4 b/0219194937_cosmos_minibench/videos/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d2fba7b97f2417485f6679dbeae0800dd353088 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_co3d_ff3d72485a9e8b1e1cc45e598dfd244f3d48a38b9f94130a0a3436a898aeaa23.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc97137ba246b769e727f492ab5bb291ec33a12c53c87666e9057a6bad65953b +size 999106 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df382cf83c1a5bae0bd0bb841e404acdfdd77db2 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_03b2e3fb03fd24c0aff0c5f90b57cb735d3ff9637aa549a2dc2f7bcdad5a040d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ce1e2178f42f2fd8b744a83306b6eeb0e2e4ec8c829443277f3a1ced56638bf +size 12844337 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..279bca8ce9605005d207f421f65b725b10da56e5 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_03cb625088d5ec191ce8667f4f6f34a17900080b36ee4b89c16833e13852f206.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4c3a5611ed5b3f6c4e9d0ae0688b883d270a0054da7537df23c67d628e9e27a +size 8787615 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..97e56a8b6790a115c98c8d4d563e1066b34410d6 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_124fd335d679c1328e7c0cd80ce486d8a62c20afae2b6470c275310b682b8093.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:805f9af46f287a8003e5a411b2fe3017b620319e2019307f28cda7bdc35bc65a +size 11028576 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aef972095564c714be68a08128b7b3411d3b004d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_13fdcb2da929d42b26bf66aef281e988a05fb3e2b87232ad13b9bd6b6f552e4e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d33764129abb41a4b628df4190ba0c828712c806c80068b82676a5f57689d07 +size 12001892 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40f493940a05151c01c584de71c502ca4e0b7372 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_219237c0e9ae17a757ec6c8abc43e5946ba680e534f76477002ba47bd2c314b7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27dff9b4ef8e04e2ba4ebd9e2fff52bb936488ef76839760c24e2f7212d7d846 +size 8255371 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea981fda489808f82f86316766359f9e71b5e916 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_225b22a8c735f5c1ea2bbb806e6d9a1abac0902fd493820f0965c03fd2a413f4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da4aca2d0a78148dd5ffe897d3fc85501eae6e20e103ec39085bb2d122be9fa0 +size 13447579 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad0284f9bb3718cf02170da70b43cf86f0727533 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_22f4385b2f47305c70569783d0383f4bc19c42718f9bfc43b92aedee2626909b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98e6b25e20f5e7840b602f7519a6de4a346bf35b304acf6d4685ba93ddfc7d7f +size 9714866 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c1967af5f38e49ed1f20546f2eff47ff94e4126f --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_24a8010495c131df0a5adee79e0a26e7ca8d5782323c270578aa808413427164.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4935124fe97b90f83fe0753d81ca0625bea3310087d7d8695d66ab293309c44 +size 4664798 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..55c1c1cc1652d3259b051d3945d2bb32c0d8f85b --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_2bc47d6e9895d593b3537d9b2e7f189b9a8604eda114442b8fab0e6fbbc8856d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9825239e3572ff6b0156f1f77facc7d600de7436e6d984cf877bc4ba6c4712b6 +size 10185460 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f73c4f9d55100cefde13126ba821cf2a94da46ad --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_2e4d41a82a8240c1c0d7205462fa436385760d1cd403d78eae7ae9cb248f9584.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e68174f44cb588cb92c7195f7d4c608e2ccf6698e3a5939415781f1ba481a568 +size 7554931 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..71733406d74c390a07ab2e48241d19823d4ef227 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_2f210d115b5875bd6a921b30137d08f30ef7b8f844b2d82e63ed23171ce9d3bd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39cc9da2fe4c1ca66a1ea2a7c3c889080b4bb324a3ea5a0c5cb836e51f5e365e +size 9162874 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95e772884f078b4f3e329fd4e6a1d355c43f6259 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_3566f129daa17ed5e4dc2b67773765d9805e563794dbdf6cd17ac3f8edf8b61b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48c8142819f2b556805daf2b1e2c4c2d8438315ea9df1455ca4ce6ac316b51be +size 10231583 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..450db2db1c0a1a97bfc6187cd0ec4d5db276d4c5 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_3b066a86b1bddaec0252192b42a09c3c315e3dea22146dc6925eba1e1f7b569f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a337e775298f094eb5c80dab6d87d7fd021fc975beda7ad383caa8cb88353b8 +size 9556889 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..590b9f42cbf426e16c8c3d2ffb30721a1bf95431 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_40ad4e9eb1e46501d03cdcbbe29dabbef4b908b2f5ea97f347664a6246cc1c18.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cfe8cdf4bc00c4c61c0ceec122ad5452fb9d5db00de65d4b40daafd0e64a8fa +size 8303242 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3a723abd12f4fb1d8206b645354a29be8349e6e7 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_4f993c63692760b0b166b0e8693713a7ec00539d7f5650211bbe6397f742cf03.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ea6d4d115c87ca877422f46c15652bcde655f5b421c0c8220d7354b82f22d2b +size 13665487 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..600e176ad49998cb6d93a4b8ae8cd65cfba4476c --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_5a47c078618098f3299850ad1bbc4f361fc4e6897cc525cf9afac6dc5bc7b4a8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e342f661314c7004ba11647aab4b3e46fa3440fe3490ef14f73cc894830dd2e +size 12508357 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1efdb777da7ed32948d860dbf2dcfac97b2d2474 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_5ae8bcd02acf6cd0b5f65b03d6adbfc55e48b32175bb430cabf2598a503b7f9b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f20d85051a2b27d8fce44e0ea22400d139e5f54549e4e344cb4970912e1fb755 +size 9802982 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5af69d96723aeef538921b1bbffc1e554ac2fc85 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_6fef07ffa088f12f4094d0fe714b7a4132ee630e62d9b5663577783d1cb493c8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93800613858865955741ef33eac5a0434cca6be643d863a05aa7bd731ee47a0a +size 12810461 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38cf50f32a4cf817a7db86f07367764975d6b965 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_74bf8aa3d43161b484c5df51374713dff593fc97e801e0de253ef6319be5d75b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8708a75eeb438bc4bcf420b2a0f81b6b01851456ae1b8e62f8fa61c410d46f75 +size 10641996 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a1f41f021ad962f57f0f573bf1c8791c39bcc1a1 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_74fba366266060f0b0b4cde61062cfc2c088a4adb4cc3d8c871995a8c71619ee.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc6c168bf8e1d5e0cdfdfb7e5eb1e493a826c2e6f5d8024ea56d47357a7e9241 +size 8018288 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af57be1fb8c467fb66166bd84f7f7a55ed86b332 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_79a83bafedafbc6b5985d320fd21c78cd4d946d1b35fe62d5e4e4d9cf8ce8167.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c5baa68b7920d8ec055cc18c406c48d8af1dac0e6a8c7276c51f5aadb252dab +size 11841294 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e4be861c5a9f06a7c714d04df2cc25bb0f18750 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_8093e8ff907550e4a3a9843e6c32ddfd147a58c2ae0213649e54356906c7709c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:119a543fe113425427fda7482bb0557d3037456f1a1ae821fbb5f86a70b07df6 +size 12242391 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2c5218dc21ad0d09357e51ce68fd41f2bd131dba --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_80ef1075466e1824929cf6b5f8b77acf2a7a0fdae07792f74e8f9905c6e07807.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a19cb27773b2eab288b4acf8f93cbc48ebd0af5cb1cc31acb968a70d7b358f89 +size 11670113 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4be687bd1214b09e6412929f28d5205405a6689 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_8118d776cf7856c5ae92a6f10ec5938e23bfc2867c10a4949b92a660637ef459.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a574daab522575e36843d220e6fe8b5d6dc867ba61c25c69970b20a3bb4fbcf7 +size 14984758 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..017392ee3eacaeecb5ea9f87d1b89e867baeede3 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_81bf2a03aadbf20bf20aa2feb18ab8b9afbd377b5be45e3792622e6e8d227a02.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4a3af97e34844cb96da4bb958cc72e7b43b544d184bb78802f821afccd19495 +size 8275095 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40295e8aa03d0c7eb6ada53192eed563721c8ced --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_8682dff2416536005d41c7d6b54cb77f517de695bcf25314f5ce43023a224188.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60c50481e5e2a8e254e28183f6d89fe06fc9aec3e1507c18369362a53ca68f6d +size 13698118 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f196bef40ec9d8a1de166a61ade3cc207aad945c --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_873c89398b526c56b09d5bc6ca701a2ed2d7a3785f2fd546f30ee4adcc5c4c2f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2f66886cfe46e6d51b9f0773bc4be82b33bc1b1db4c50f496460b08884b7767 +size 10363473 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..90fa20fc4ae39aa590ef38a081b82b277da5e80d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_8879cecf92e13363ba367f06ed1f8ff15152ace4291130f9a4daecd90aace40f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:398902052efce262c555f3a110ecebafbbb5aac08fcad0be755db3aba4348839 +size 9320363 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..115461bcadf14b561cb5b26fe8d4749d081b88c7 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_8b2495421084723ea2151e09fbb096adf682f541eb2a42c26ce57578b56d8a97.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8299a4cc39cc703732bf5614c59414de029feefd1af4d3ccb8068a2932faf6f3 +size 9030814 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a565e280124087da6b99e58cc7937101b5b50878 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_994729a47e8c422960439771fbe1d8521844636ffd23c39802813d9b967c4068.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a5bcdb859e4112a5ba338db2bbecba55f706319b726e67313af057d0c9e2bbd +size 11310531 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..36f35a8535e82867967d1ef0f88284d75ccf6862 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_9f7018c35daf35b3c3c49ba9c99976d06f9dddb0a3093138abcf62f3e7d94b5d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa105d84862f1c098ae38cd4e3215c02455e1b66b7feee35390dfebec472ad00 +size 13604857 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..10dbc507b0683bc5a5202c85d422b0c7b231efe6 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_bafe35be4993582f4897ba101902ddf424b3c6c0e2b98d8ca8b40ba2814c535c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:229cd5fd3916474a175783a5cdf7b094774d244788c582e1a1a13628270ee1a8 +size 11958075 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c1762ddda3d57046ea7e74cf1657f4f80580265 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_bd3da6c5820ac1c1fcf0149c3e952c04be4d3a23595f19d269df09107ac3a959.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae0ecfe2a0a93cf3428ff57a28a852d5bd3691374e4c66c926c6c8b22e5f63ae +size 13108649 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70815da64caa755f4cdc651a130033a548871c81 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_bde60503f811435bd7d0eb9034ea71c30ed56055a881f426fec7b53e07d5bd2b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16c39d1fe808c208d4d52ac3d0ea6f78ce8bda980ef719370ef6535b83ad839c +size 13786693 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..50821c2baec2639e823133a333997e1d8ded567a --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_cc40635ee536377bb36fbea01fce5d2cd1c3e3bf56ba8299a84f66a3b9293b7a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bd76eeae1ae783260ec3330a75824fef215c4a9832aeabc8e3ad195949f4419 +size 9057428 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6178d8c192f51f1391bd0fe1e26731c3e6d42137 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_ce71acc3017d7df0b85516e8ac7b87ce82a552a4840df3c0bc6d51a5594f3c50.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ac87c99bb0b2cbd1dde7b170766d24ffb101272cbc3eb725b1e959d6cb132c8 +size 10891058 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b82725bc99df88db4e661ee38860667f256c8038 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_d75f9a607ebfec618e585d62b591fbedf78d7ef3af6126c6f7d582ea03d03336.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7af381a6e033c4d681f902a3e4e775a42a0d94ff242e1e409d5e6a7879cc7d87 +size 10232131 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9567760233854ed98ca4a5f4d377fe8365112178 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_d99d4c9126a46474b90ad4b2565d9daaf529387572ccdc388e3883083b66391a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00061f65bda5eecd3becb4fe16145debe4d6f3d7594a3dc022d5ee0d1f94ff64 +size 12667163 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a28ff581939e3a53d5c6a9e485e50d597610a295 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_d9be64a8fb23fe52670920fbd06b6e587b4fa8d15cfa1ace227cbd6c199d76ea.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1e99e1ef1995284575c6d2ce5b80b11883a0a1f1e4c4e739a32e6f12e061878 +size 10276803 diff --git a/0219194937_cosmos_minibench/videos/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580.mp4 b/0219194937_cosmos_minibench/videos/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bdf7217862fa1f2d4ababcd04c4365d198419cd3 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_dl3dv_fa290517f6f5102ed0544020b826762485dc7fa96f8b49c408d7d54c96dd1580.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a07fe9ce513fdc308e96de8fa369add20c3c0d83ff9be0e3de32a4c116dc3fa8 +size 11618720 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22ca4317dc9c6d26cae28dd891b17a9269f0b892 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_051c290c81ddd213d3964c5b8d93c2afa9757d56039bf913c3fe7f8ce2b04104.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25037405f1a7a6bfac62b95459198b767b15d093f8143d5be4deef4728df7e36 +size 6186067 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..69ff8201ba380b1bfd6f5a6f8365aba0f1672ebe --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_0e38179170e72cbb4febb9643bea1d5d4fdae80e2d2fe4b2f672af8a0e22e819.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03d988fa5093a5563aa4a01bbd54ba81070194364eccab91d9637dcb452713e2 +size 8517145 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac11683f6c015b32b1275e05f8b4431e3ec014df --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_1061d6b7f9596d5964e7d99b069c4aef787f1e8d7e4c47bfb8f61eb4d429f1a8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a2723bc345a55f5cfc037013c0444a7ab4f842cbac5f0de7f9ce483bb91afb5 +size 4877369 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b66f0225f7e7101577a88fc8468e5d8769ea6c06 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_1147c89022886daf803c41c430dfe731eee63e74cbf95e8ea397989e5c7c1706.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7c21a8997f0625945c9265dc89d76955533e85e54ad828a3b1748f16c7ef318 +size 6449006 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eec57fc8db9da6f595a656f464aab513a91c6b9f --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_135f9370e5406e825ec0d57c72a8c492c9375fab35e00fa8f14e34c043e3add0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9348f8df40e63494385fe2ea09ce9ef403d6bc053d57b6d759a50301aab32e8e +size 4767043 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..574669ceb23a2ef70ed4052db6a86ecfa65f4f2b --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_1d69e5013813ce3320ac2034795209e67e1c206cd96cd975de42b062baf196f5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c390f50be443e4cc8e62eb1265dcc6e93265c3ef803830ac1243d12b8b34686 +size 855580 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..391852d758f4d37898ef8f81f9e337b77d7fba96 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_22c7af9a857cdb269ebc9b5dc0366fed446de39752b38e6aa8c5ee595476ba6d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79704f6e60e9df8e28904ace7e252b9134de5c3f39d499caf02db42ed6607f10 +size 7965360 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf680a48cd4963215c805a3382c7ee92ff97fece --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_2744130b53b06f32d36a6fa01295c2e44f305f7c298b006d46429ef376d73646.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b1a1564ef5d7105d9614f92d9a1d4ae3c9618bdc302f974dec439f4a4144c10 +size 4708321 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..461594754162b9c00585c5c6ff953fee3f0a88b3 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_2ec44997b19ce06ecf8d166765d0eff597461a97e137c93042b7526486efe8e6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b5cb714581c6938b88f56a304ec5aa58979c15dc5d3cb01958de53ecacb8dc0 +size 5901342 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1f7189df243eb7b77e08d0f200cd948a24ce0eda --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_3049f2855ee59a9b58db0d795e9a127e7da0c28ee7b582bdaed48b3d8a5be129.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c452c0410b929f4422cbc4da94f5a25bd3ec1a0cff36e238bc09fb20903d475 +size 7992343 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63913a32e40581db01d10971b4010de65880390d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_3533efdc87f08436b79ff62ba8e9277fc497952e4e9157808b21caac2b81d009.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b090824eab12878fa147d0292e39e93300275bd364c2bd827b14b77721b28de +size 8494320 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9471f51e96082e1c2ed3f0513524d6131bd24096 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_3ef5d21a0000261a627f1c637e2a12462d332bfa3eba276c571a79b7ccd706a4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a853eb15e7f210fab33254c8bc1369ff382d3f9f57d467c0fe93bdaa843e415a +size 6452456 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a55d1d860595202ddf2a954fe373bfa63fba76a2 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_47473f9b5638b998bfc9addb37fed7439fca0ebddb667d220b21101d30ecb2c9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28409059a269b32078f8765c33140207c3038f8249ecf95581a3428686370021 +size 7656023 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a83275fd6610becff2a5262b0e5c07551696cc6d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_47935fd69062172c4ef29092c388b9a53956cb694e79fb25eee2db18b6b9fe6c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95c5083b3a6c0647c123c79e518939d935864467b718691c8447a95244530b58 +size 6107516 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1d49da6225e5e5e7f1b7dd2a15cad5ab93f0b493 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_55cfab60ec7dc9ad82db3c745d2188c60cdadbf357bb184838bc1f7f555fd2d3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df83b21905dc73fa1a81cd6834b7a0766e4f2a530fb969d58a3ca37ad1457e14 +size 8005107 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bdaf3afb8a7e5d14c1a1e31cb48c94ca83c074c3 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_58abbff1b36e3441e5c584e340b34cdcca820bb4b081c63a8855e8cb8577f4cc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13ac88393b0e9fc70d074f2ecc691a7ae3f5e690b0f622da9a373baada58828b +size 5711455 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87c3f45121571afe3ca66e5207fbb1114940bbb8 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_61b6cc3d37d74449639b53de26e8805fefc6dcaab4b1bb1dcf6398a8ca89cf2a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0f1b4e9d23cc9c2dbc47a4fd93a970c6e213d29b5401f1e261f12798896ed03 +size 7477912 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d2f9dd19448e18739ef6f2eedffb3984225171d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_688765b9178f4166e2a67f1f775eff23bde7ed9f78be3174db04d52355c5afbe.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:906ddac34d0aea6479061840d369e01a24204e9c1517f74e1d62d32b9bc18b2e +size 7039179 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca395fc1bb33e7e07238c505f3ec9a246966b76d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_695418a700aaea62bd90ad615728134fd8be8569db8371a3a2181b33905f7a9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca989d75ab7a80a174748c1b01523034e7ee96ac26c11bc36710952fcbbc27b5 +size 8341617 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8d9cb2a2e753e0c4306ec6adeee354057fa85823 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_6b375c3c386cde79052b877b17ce0428d834d0753a8e94bbe737698ea4d1a73c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90c01bfa61a295371e44f660909e85c79ebc678cce685a10f39a79393a25a189 +size 7143749 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5ae366d3d060c870d3c11320edc01a278e1222d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_6e6d8ac0bf3645033f8d5d2908cdcf0082be447c49118a75867d4aaf4e018872.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b089d0ac8a7efce61cae7999122a10930436f8a9b82198c4030213c30f87175 +size 7330640 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f23cc77b262bb92b28355e76b7b9a3670b1ecfa9 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_73af1cc3ad4f46e3d206882d0fa4424e7ba9e03e61c57e938738fc117989da52.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d87d0f532edead9a72086e38f187524bc9c336fd0aa1bc1f84a1a7fab1b84e75 +size 9177040 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4a28fd0a4a230f902c9be1b4a98768c938bc0cc0 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_750b40d7ee2b54068d69de8804298a484f32f8d972c347369a147d02132ce7d8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1622214beecaef520477eddc1afbb1dea9b1ee4617ce73d82105495c0d54dd8 +size 6231528 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5382c6b4fa11173b074a8b3575b8dd7bdb154529 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_75dd2f45336cdc165e62783f0ab5693f62ffac38de9cbb5e50e54b51a0b312bb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac9c6a968f4cf399c86939d11505e033395c387be0e16a698fa3c14dd251c9a +size 6762446 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1f205cd4b5920f46333b788cda12bd3b9fa74c89 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_7ea535042c7a214315643063edccbbf3b59e8dfcdffd58a2a88bbea1927c8940.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97176eb1c19a079b8c05aba5e5acd8f156c7a3b417bdcbc8bcd6fec3902b99be +size 7725167 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a201106a2e799efe3ce2d2e3f17bcbddf330b09f --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_8b37a20865353e27fbd48d32e33071f6d76850b31281b17436ff944873375ac2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14b62214f256d3a77e83c34752f5287e7935f5ee678da6c637eef59c399005df +size 5736226 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ddc8b20cf19a684e86f600eec5a3087177864a1c --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_8b3c4e0f11ea369db0846369c12fc053d6aa45c4eb0106826cb7887fd824ad32.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5aa8f55ac28535e98eebc86ad57dc0b0c167c150fe01cf4c34fff464b8b7d2b +size 6128173 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e5ffbb23fa5ac6afdcba80d32ec422fe6eedd777 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_95631f78843ae8063f3ecbbc1b8d5cabc0ef1c880069b6330191a492c65611ac.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc824e37be8ed421adca5ffab2449e514ed6fb9efc2fc4b1d2d547538fe7b52c +size 5878767 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41e2b7a30a860f5811684a5ab48446f72516f25b --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_9827146f4dc3fdc6b3940eee5135b8b969cd5f0939e309801722eae85728ab39.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:736e25e6b0919a71c775255207d72d1d832488c0c53c1c518858ccae6819b38f +size 1870629 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bbbd13d1c8f6ca2c738db376831883f99ae663d9 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_98716ade815283408a38fc2da088e7b670b0bafeec408a2281b078ba38495bb0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:428ad20f9eaec9bf5127765601fa265de2b44942a88e0a7858b41138a68c6029 +size 10002717 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..706ac9e60477d7b53a071592c5a40cee93e01c9f --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_9e47b70f0d0d243a3500d03036d4b50b2dcfa06fdba8301ad9a9cb5287f4ee9e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba422e3b2129423f4edb4c3c0d0365ae54c65b5b6473de3f67fa0d85ac57c716 +size 9877048 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aab842799e1e3f73c4d1afccc8d7768459827e25 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_a99f6b126a508cb70d4d4fbc99e5d4eeb021140ea58af7811e0f591f1a1dcc1c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30671f585656ea73aa3fffe595de016a2e71234dffaa2891727fb45317e23017 +size 7053436 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..55946aced8df3ca4c79dfd0093d3fe06e2f42061 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_c29e77914bc59d4173fdf8b1ac20fd381f44dcc6b94e7e6cf7c488bed830cebf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bebdb0ed14f2f2312158abcde730403cb4ff763c87ad08beb37b75e2e6c743b1 +size 11166110 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3c422dd26f50ed12e1fb56aea3358c5678003fd4 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_c3ec7b6759d93ea72672328422f1f0c0ab27c03beeef370c9ad54809e57f8622.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9032faa5ca9ba7e3dd87fa393d9a74599a2d645530ef794d1ebe1d1ad99f556e +size 1521716 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e405c14c9c8463217665feb33b8f75315e41d41 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_c880cea11b7c826d8c29b8313f391036d4ff335c583f12d7896e8365b8c3f328.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70a8056926fc9ceaf3dfcace6d34147ac0f6cbe103e481c3864cc65e6492610c +size 7034640 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48ab9222081bcef70de4e28778cdd0a1f5bdf05a --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_e19832a424258cf3071bd7f4d89b822586cce68c1de7fd08586d1907ad3abf8d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e3bd8c7c73d5254916296ba7ea9c47ac8d4607df0e6d9ca132d6f29b7cebb54 +size 4627953 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61b6ffede64ce70a4d142a91228911b008491179 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_eb6e8c7ae0644030bcd80c015e1f129700278ce8a03cf4f117e12e07c3df13b3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0341f1d6abb697d78c78f0a88dc5863fe6c2d21de87cb67f2431086f16b87bd +size 6682082 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78c09930bbde2e377a6269f70cacb5962ec7b16a --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_eba42f272d07b335611b5a5f8ce09ef4a88aa036322bbf24a5e0e357ed0f59b0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1580b050ae43fac251ef27a0f8ba7750cd021f3826528d0dc9cb2f88f3f1ab99 +size 12681008 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e79740eb21aa2c1b81c5d95db3aab2f8f6178f1 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_fa64a873627f41b6a08cf9da47bbe887483dd9069dd0b8d5b9969647b78ddafc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4628536c20a1dbdc00d07ba8556cdd974a10fe2678df600c18ad6290d13a4849 +size 6415721 diff --git a/0219194937_cosmos_minibench/videos/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d.mp4 b/0219194937_cosmos_minibench/videos/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..feca241028d11c85c539f48e37604d43026816be --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_egoschema_ffbd754abb183449e593a6dadd3fb4dae99ab1e003e0f66a5937514b7d9cbb5d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f44320983e050ea4cddef3bf07f2bd0891cca538a128451927165a1dbcf16461 +size 7391785 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..795b9c146a8d3d7ace92c3669869f83068106c08 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_04d78dbd04b80da31ab5534936f52686a7ffa2af9997c7999496ef4aae1dbf41.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baed0757c270c5f479f2d914de1465c6e5b97d4c8b9f1b7ea0eee2489db4ef1a +size 5145693 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ce9f439895aea51affee4956eb81f949de26a6d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_079f6efe68b1fb34517ef78bbb7e0bb066477fc1f5eae5cdbcc9b14ba023e2b7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abc9d76c24d67ca65a9cea658949e348bff8c707d5d538f4ab1a403f263dc630 +size 3874792 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ed1f8a962a0ce4673a3fe6a132f36d5a8d3a6ab --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_123bf0e9ebe6b06efc7a9d81637b6990353e5fc637a15463577b0567bdaad0fd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:192cbd7482be811fec34444eed130993b239c0179e12ddd8503d8cb864d9c2fb +size 1412789 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..256453f27841a155acd818e685ec8837ccccd2f1 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_1f35e1c32209f2acdcec26a4e560e1cc7e1fdaad59f891bc03f2b157226c771d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81e1bc4cd8f5be1dd30dd1bdfa10f106752276d307d38fc1e4dbb21071a7126f +size 6396149 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..082b88b0c3ea89359c3ed8c5bc99b9fbd5a49354 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_3417154e322ba78411000363b05ee7409b73a31e7af65e5968a1908e1e84537a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0165b96a5f9f4261b308d6bbcddcb1fe3401b1fed38bdfa7b9fd533f7ddea7fe +size 6135144 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..378d47bb6fcf994ee319c5454ee28bc69b8c4524 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_3aea5f921baa2797ef215af691b392f87fcf39680db2bd1c50b7838614dc54f1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1cda63db0bdf6f27706cd381bc1c32026e41ed70483631a2a12573850cf9428 +size 2295074 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..089c8aedbfea63329d9d9e8d020947e8cfc64d02 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_3b1b26c1d1c5783d72f01570a0b042df272b9f9164f2994c4195efea46483da0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:146f905f6270f2d96b78c5d744254e95922876b36c1944e5f97d45c10425faaf +size 7954241 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..16cf7519fee0af7d1b74741423ec70e0b90b36aa --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_44d6e3ece41c19334478d27f8e5427e1aab2ccac195026cc492ea47bf6b0cec2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:115841bc2aa3aad50c84db45e3218fa5e2ae2fb8d0a9e9c2cb9b5863f53d58ba +size 7955180 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..60e8ea86a51ce00d59f3297fa0fbd0a3e07ae2bb --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_45633dd44b52667f482c9693f9160021a4a837b55a4b940563d9aeacec01cf15.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96ad463e30d6a2052e16a57b88ae4659787e57fe21c89927f32b33f93a2a5856 +size 6578074 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3caf1c18f2d1dce02955df9124a232a1a59e5322 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_482a8459664c56747f22f8458aa38f09a449a4d63ac60817cb176673cbc40f5c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad393d7a69f841f59174c60fe88743b79baa39f49a8419a5ead633dcee290033 +size 4296171 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0e076c88cba630a030ceaf90296397bb0ca39bb --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_4e5e1c0db89294099c90db5e79182763b7289b0501c9f4b87ccec06187a22c53.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fcff749539e572903b93453ef689781c256934b5e5e5602bc34d32df66d4b7e +size 1224089 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5edb89790043ed74d03dc13445e04728e7737ce4 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_56494681253156a68d48e6373aaf1f88e76740d395dde7f74b6643da8a46fb7d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a69519e65c1cb5041193140d807ad4afbc1d4434687f9a5eabeae873207c68d +size 7289824 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..725ab3a47da489bdfaaf8d4e5667b4412005b1ed --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_57a4cd658dd761e08cf3f8e32d2a6e3fa828be08abee3171a21043e741a8449e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c7b6b6fce1244585162d189fc303d90b36b88f57911857113361f5fa4b931bb +size 5705567 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..80fb9eb9c8333be6968eaf8bdcd8a191c0f5163a --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_57e0cb96d5bbab5d8e4241dedfd509901e1e49b4b16d071dd2c83d6a28821067.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4202db57001903ba53df5d1d116fb9c9ec4270f94b2b3a0983c5cb0d83b2a62 +size 1974200 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f5148400988fbde6e7b325414a34f171859c95d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_624afc154dfb1dd55409d8cec4cc3435f0a00bacb0dfd05fe151dac12b936f66.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:707672c49781e6eaa9ef28e690de694a7becc172f61377574f8def744d5386a1 +size 4349467 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef64e0d07dbbe4032ab6cf496ff415b955d7d3c2 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_648172ff667b555bdbe841c37896d863b957129f8ba0f72df65ef44490d1d632.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24114038f91dfd6b54fc2112ec6a9823e28822cd6792c9725bc28d2f07db89bf +size 3040987 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4343c61eab1eb92457c36286c7d8c587af1ef3c5 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_6a7d09a3add8ca9b49569c206c1d279510f527b473e6f3b976f41f11271da841.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdf2d45503acdddeb9514b73dee8b4206ec9fec589c7a8446961abb3be66ba18 +size 4289487 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..218512d476b89a5f1f61d0720adfb29fa3200ed7 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_6b0096237a984d01469ed6f68759743e35779783fcec334e0a0793ed682ec9b2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73d1e3ec40c6ed243d703723a292e23338e36d47229381191884aaaa405d4ede +size 8999048 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c029bd9c8256c96367a636827a68498d6fd7cb36 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_6b9b1a8e216fab48de8cd19e4dc64623938d6ba960de15aa358d749b58cf4ad1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:831172005a51377d169f0b5631402173473bc0cbc4c643da385fe5ec35ee009d +size 4830866 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4c78e2c4e03a329259f5633d9e4edf987ae022b --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_73ed464ba39c63e4ff1de18c3d429a351571ef0549c9012e3f4a6cfad0f0eef5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b158c173219487212664421f17b03a433318093306ecd7067268ada48e0416ee +size 4919997 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2749f8bcead5eba69f0523350464ae65f70290d7 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_793e3f6a347e2672d701b7ba42d6ee1450191f0fb174156790ee3dd2bdca130d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c534feb4ad351a460bce3c2c272c01008418a5beb3ad9248d4cc5b59e1a4cf70 +size 5140913 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f0f9e253dfa9e28b94322d4075a2648db8270abb --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_7b09258d7c70665277b392483364194c07b779417e8db58cdc07986f8784de06.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a96154534983da36808c302e3a37e77d063b7efa5d63ae0622de37179839e218 +size 8621917 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..046a02993403d921b2e545ab28642b49564e966e --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_852002d435de2eb08124f2db92b982743836a02733476284f3befcc4563f1a6c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cf291e824e8e03d7d0812989ec4bb6ebc8b0de389c9a31fbae6fe486aed494c +size 6066229 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..76597c1709f0180c96a7e34e5e6ef3417d90b979 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_8ae37cf1302690f05069bc82d5cc4d8502f32b3ea4fe097764698fe50147f2be.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8665d6f87bed0150419eca34f19c0139d661bbd94ed13ef0d0f255f856bac16f +size 4518309 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..acc824364770693a864d17f818b76df1ae834f4d --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_905900711f77b59bbbca83e0d71469a935b4f6d646f82da062039c5454f57287.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2cba1b3eca9b8cd93cf5a3e8bc2c7ac1be993207429d4c99e6cb24db917dc8d +size 2528549 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c869a1de2c13bcce289e368eeb081df3af54dae1 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_909850a1b95ce39b4d7b9a0fde693d5aa4c72ea05412eaef5ad854bdcaaa06d5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec5fb4cc790d26b501ac0596c15b30707cd27cc63a701fa27952bbe5bb794dd5 +size 4471109 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e407c947fada82e90f2efc5c5999b14cd5f04049 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_989d76420e9f2b2445c65bc49854a7061552e401437784233ed7d8d19bb3d0b8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd8e5591a7ee5c97608ee757b87d60c167201ec3dc84c40d55b6f3c731909111 +size 430958 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40a284faebafe274418cc8326bc994ca5bc87167 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_a4bb26585db8a889039fb506d2bb4951fb4804d609e03426372abe70276bc212.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0be53d846d5d75a94eeaddbd13b5e777dc94d27eb03bd010e1421f71623cdced +size 1206034 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..890582670e21567fd8f64d62cdf470ae37fecf75 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_b23b88de349c55e07f0d678a79e0f5b777848da7a12184fbad56a6e3931bc960.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b13f716ce1bf2210090d3903c98d6f33f226d83fd3b40ba1a5c9ac1e3ead41a7 +size 5599249 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61db9f44d79e8d01ec478698f1deb534b46af4f6 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_be4fb95cc857fd79d91412b4ec979e8e80042522930bd26b6cb3adccdff3b38f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94630a4e53b8b92215b84daf62edaee8a958e756075119f65cfd7a466d7dc056 +size 939628 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..28c877eb594cb7801a8969bb79e78a3b0c576cad --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_c83b792f5f5f7e9b9a72bc873c8fcd7b6bf3351797df0784323181678f56c8ca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ded2599624ca25b3d64514e41e05f3474bcc285de40b15c4d7f6bf65e54183a5 +size 362888 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d494bebdad4278dd50d1d1a6eb742247bd59c0bd --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_caa2aee9b61501d66e346b858f008bf700fc70c3b04c2bdb440b9474109c1290.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c160b8560bb21c4ab5278018e392dbb0cc12ea36fe43a87071037a497bb6cd5d +size 5329740 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73784ba702422f5e3e1cb622caddede4342f2498 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_d5e1d57e200d8bffefcba4adcc12e630e32e76d77bcca75e6df4b07bec375183.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e48b215aca4fd1d53e7dfbb836818a0220bcd8322d184e7757916418cd64260 +size 6087709 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..295a8e09e50db9ae9b5c6d31027617082c612870 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_d7ec31e6ab235b2d6efcc5342395ff3195a4a0e059021112a46d19491ef0e529.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4237a7af9eac8f486df38376325f7942ef78198babc35eac2889506c89a7f523 +size 410839 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1933f765b589c1593e92f205a3e7261ee51cd8d9 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_e49eaf7fb50ac654a3b5f1f1ec97b36bce3174aa741015475573ce357ca8bb5d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0b55040fbe56fe30204477c1a6bde0d4f4cf01c8e19a7fad65a248f3909646 +size 3520763 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..798eaa0e8e2d1a3e1ff14f17b91eedf274525975 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_e4d23edb0f34bdd2c5f7282819eec3d83ff0a11016abed20074b8dfdd892b67e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:627a1eaa71e00334f6cf8106bb1a5a14544e963a05fd4ba836a7edb028523d30 +size 4016097 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dd6d0356732ec713dc52da0f8af715337966c889 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_e5642679730817cfe6c3e15a3c61975ccfa96da65b26bbdef38218edf68b504d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:853e22e1c3e5aa6178f290faa5c8f66a09f187036e9052dc2b24b4d4177f153d +size 5523678 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cca17338c4877f1cc6020138275c6a212ec7b64 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_eaa7f4eb002d54e263471f4da094e428caad90be2dbb734f85ee17a1bf63b168.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9529691805e92767e57867c722288c13ac586181f3ffd2cc944f7217f7a10b05 +size 8751213 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63117c457a42b572b7d160d6fb317018d2d2c670 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_ede8b958a02c1c954adf48b32693b09368ea497223b4518453c7c800a2a4b9b5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7701c9f9b79096984380edc245e8798e50c03e179adbe41867367dcd4554d81a +size 1178370 diff --git a/0219194937_cosmos_minibench/videos/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d.mp4 b/0219194937_cosmos_minibench/videos/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..183bdaf1742a3a8fb4c712fa6631f17b094647f3 --- /dev/null +++ b/0219194937_cosmos_minibench/videos/mem_openhumanvid_ef35121e8c5639c183dfe54ae49a58ddf0d2f96fa015b354ba57352b531a1f6d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:590a0a39999b1f405cf8960dbe72b19a7e7e42e87611cc605f36cb090a56f147 +size 5525110